From 8dfa8d85500ed5157f8680159f8694d33540f091 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 16 Oct 2024 21:29:29 +0600 Subject: [PATCH 01/40] Move V0InitialJobRequest to bottom of file (for easier review) --- .../mv_protocol/validator_requests.py | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/compute_horde/compute_horde/mv_protocol/validator_requests.py b/compute_horde/compute_horde/mv_protocol/validator_requests.py index dc25891b1..0bb4e6dec 100644 --- a/compute_horde/compute_horde/mv_protocol/validator_requests.py +++ b/compute_horde/compute_horde/mv_protocol/validator_requests.py @@ -50,21 +50,6 @@ def blob_for_signing(self): return self.payload.blob_for_signing() -class V0InitialJobRequest(BaseValidatorRequest, JobMixin): - message_type: RequestType = RequestType.V0InitialJobRequest - executor_class: ExecutorClass | None = None - base_docker_image_name: str | None = None - timeout_seconds: int | None = None - volume: Volume | None = None - volume_type: VolumeType | None = None - - @model_validator(mode="after") - def validate_volume_or_volume_type(self) -> Self: - if bool(self.volume) and bool(self.volume_type): - raise ValueError("Expected either `volume` or `volume_type`, got both") - return self - - class V0JobRequest(BaseValidatorRequest, JobMixin): message_type: RequestType = RequestType.V0JobRequest executor_class: ExecutorClass | None = None @@ -146,3 +131,18 @@ class V0JobStartedReceiptRequest(BaseValidatorRequest): def blob_for_signing(self): return self.payload.blob_for_signing() + + +class V0InitialJobRequest(BaseValidatorRequest, JobMixin): + message_type: RequestType = RequestType.V0InitialJobRequest + executor_class: ExecutorClass | None = None + base_docker_image_name: str | None = None + timeout_seconds: int | None = None + volume: Volume | None = None + volume_type: VolumeType | None = None + + @model_validator(mode="after") + def validate_volume_or_volume_type(self) -> Self: + if bool(self.volume) and bool(self.volume_type): + raise ValueError("Expected either `volume` or `volume_type`, got both") + return self From 3f28a4c22b3ab4651afc64e7b58b281279546cc3 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Thu, 17 Oct 2024 02:05:44 +0600 Subject: [PATCH 02/40] Allow sending job started receipt with initial job request --- .../compute_horde/mv_protocol/validator_requests.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/compute_horde/compute_horde/mv_protocol/validator_requests.py b/compute_horde/compute_horde/mv_protocol/validator_requests.py index 0bb4e6dec..5cf368860 100644 --- a/compute_horde/compute_horde/mv_protocol/validator_requests.py +++ b/compute_horde/compute_horde/mv_protocol/validator_requests.py @@ -116,10 +116,11 @@ def blob_for_signing(self): class JobStartedReceiptPayload(ReceiptPayload): executor_class: ExecutorClass - time_accepted: datetime.datetime + time_accepted: datetime.datetime | None max_timeout: int # seconds + ttl: int | None = None # seconds - @field_serializer("time_accepted") + @field_serializer("time_accepted", when_used="unless-none") def serialize_dt(self, dt: datetime.datetime, _info): return dt.isoformat() @@ -141,6 +142,9 @@ class V0InitialJobRequest(BaseValidatorRequest, JobMixin): volume: Volume | None = None volume_type: VolumeType | None = None + job_started_receipt_payload: JobStartedReceiptPayload | None = None + job_started_receipt_signature: str | None = None + @model_validator(mode="after") def validate_volume_or_volume_type(self) -> Self: if bool(self.volume) and bool(self.volume_type): From 972ba4bc975ed7b6208b8060ae53787053b4b087 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Thu, 17 Oct 2024 04:01:49 +0600 Subject: [PATCH 03/40] Add timestamp field to all receipts --- .../compute_horde/mv_protocol/validator_requests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/compute_horde/compute_horde/mv_protocol/validator_requests.py b/compute_horde/compute_horde/mv_protocol/validator_requests.py index 5cf368860..1cc65d50d 100644 --- a/compute_horde/compute_horde/mv_protocol/validator_requests.py +++ b/compute_horde/compute_horde/mv_protocol/validator_requests.py @@ -81,6 +81,11 @@ class ReceiptPayload(pydantic.BaseModel): job_uuid: str miner_hotkey: str validator_hotkey: str + timestamp: datetime.datetime # when the receipt was generated + + @field_serializer("timestamp") + def serialize_timestamp(self, dt: datetime.datetime, _info): + return dt.isoformat() def blob_for_signing(self): # pydantic v2 does not support sort_keys anymore. From b40f833dac9a4be74ab48a92043ddef933af84a4 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Thu, 17 Oct 2024 04:02:44 +0600 Subject: [PATCH 04/40] Handle empty string cases for None --- .../mv_protocol/validator_requests.py | 8 ++++---- compute_horde/compute_horde/utils.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/compute_horde/compute_horde/mv_protocol/validator_requests.py b/compute_horde/compute_horde/mv_protocol/validator_requests.py index 1cc65d50d..85a2017a1 100644 --- a/compute_horde/compute_horde/mv_protocol/validator_requests.py +++ b/compute_horde/compute_horde/mv_protocol/validator_requests.py @@ -2,7 +2,7 @@ import enum import json import re -from typing import Self +from typing import Annotated, Self import pydantic from pydantic import field_serializer, model_validator @@ -12,7 +12,7 @@ from ..base.volume import Volume, VolumeType from ..base_requests import BaseRequest, JobMixin from ..executor_class import ExecutorClass -from ..utils import MachineSpecs, _json_dumps_default +from ..utils import MachineSpecs, _json_dumps_default, empty_string_none SAFE_DOMAIN_REGEX = re.compile(r".*") @@ -121,9 +121,9 @@ def blob_for_signing(self): class JobStartedReceiptPayload(ReceiptPayload): executor_class: ExecutorClass - time_accepted: datetime.datetime | None + time_accepted: Annotated[datetime.datetime | None, empty_string_none] max_timeout: int # seconds - ttl: int | None = None # seconds + ttl: Annotated[int | None, empty_string_none] = None # seconds @field_serializer("time_accepted", when_used="unless-none") def serialize_dt(self, dt: datetime.datetime, _info): diff --git a/compute_horde/compute_horde/utils.py b/compute_horde/compute_horde/utils.py index 1b95e343e..60d6d71c8 100644 --- a/compute_horde/compute_horde/utils.py +++ b/compute_horde/compute_horde/utils.py @@ -3,6 +3,7 @@ import bittensor import pydantic +from pydantic import BeforeValidator from substrateinterface.exceptions import SubstrateRequestException if TYPE_CHECKING: @@ -71,3 +72,16 @@ def time_left(self): if self.timeout is None: raise ValueError("timeout was not specified") return self.timeout - self.passed_time() + + +def _empty_string_none(value: Any) -> Any: + """ + Converts value to None if it is empty-string, otherwise returns the same value. + Intended to be used with pydantic validators. + """ + if value == "": + return None + return value + + +empty_string_none = BeforeValidator(_empty_string_none) From 24be520908391be1cc3fdb0790a737f825136820 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Thu, 17 Oct 2024 04:15:03 +0600 Subject: [PATCH 05/40] Add JobStillRunningReceipt --- .../mv_protocol/validator_requests.py | 14 ++++++++++++++ compute_horde/compute_horde/receipts/schemas.py | 4 +++- compute_horde/compute_horde/receipts/transfer.py | 15 ++++++++++++++- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/compute_horde/compute_horde/mv_protocol/validator_requests.py b/compute_horde/compute_horde/mv_protocol/validator_requests.py index 85a2017a1..ce6344236 100644 --- a/compute_horde/compute_horde/mv_protocol/validator_requests.py +++ b/compute_horde/compute_horde/mv_protocol/validator_requests.py @@ -24,6 +24,7 @@ class RequestType(enum.Enum): V0JobRequest = "V0JobRequest" V0JobFinishedReceiptRequest = "V0JobFinishedReceiptRequest" V0JobStartedReceiptRequest = "V0JobStartedReceiptRequest" + V0JobStillRunningReceiptRequest = "V0JobStillRunningReceiptRequest" GenericError = "GenericError" @@ -155,3 +156,16 @@ def validate_volume_or_volume_type(self) -> Self: if bool(self.volume) and bool(self.volume_type): raise ValueError("Expected either `volume` or `volume_type`, got both") return self + + +class JobStillRunningReceiptPayload(ReceiptPayload): + pass + + +class V0JobStillRunningReceiptRequest(BaseValidatorRequest): + message_type: RequestType = RequestType.V0JobStillRunningReceiptRequest + payload: JobStillRunningReceiptPayload + signature: str + + def blob_for_signing(self): + return self.payload.blob_for_signing() diff --git a/compute_horde/compute_horde/receipts/schemas.py b/compute_horde/compute_horde/receipts/schemas.py index 7d8cfc94d..af74ea5df 100644 --- a/compute_horde/compute_horde/receipts/schemas.py +++ b/compute_horde/compute_horde/receipts/schemas.py @@ -7,6 +7,7 @@ from compute_horde.mv_protocol.validator_requests import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, + JobStillRunningReceiptPayload, ) logger = logging.getLogger(__name__) @@ -15,10 +16,11 @@ class ReceiptType(enum.Enum): JobStartedReceipt = "JobStartedReceipt" JobFinishedReceipt = "JobFinishedReceipt" + JobStillRunningReceipt = "JobStillRunningReceipt" class Receipt(pydantic.BaseModel): - payload: JobStartedReceiptPayload | JobFinishedReceiptPayload + payload: JobStartedReceiptPayload | JobFinishedReceiptPayload | JobStillRunningReceiptPayload validator_signature: str miner_signature: str diff --git a/compute_horde/compute_horde/receipts/transfer.py b/compute_horde/compute_horde/receipts/transfer.py index 071a468d2..58013eec1 100644 --- a/compute_horde/compute_horde/receipts/transfer.py +++ b/compute_horde/compute_horde/receipts/transfer.py @@ -13,6 +13,7 @@ from compute_horde.mv_protocol.validator_requests import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, + JobStillRunningReceiptPayload, ) from compute_horde.receipts.schemas import Receipt, ReceiptType @@ -43,7 +44,11 @@ def get_miner_receipts(hotkey: str, ip: str, port: int) -> list[Receipt]: for raw_receipt in csv_reader: try: receipt_type = ReceiptType(raw_receipt["type"]) - receipt_payload: JobStartedReceiptPayload | JobFinishedReceiptPayload + receipt_payload: ( + JobStartedReceiptPayload + | JobFinishedReceiptPayload + | JobStillRunningReceiptPayload + ) match receipt_type: case ReceiptType.JobStartedReceipt: @@ -70,6 +75,14 @@ def get_miner_receipts(hotkey: str, ip: str, port: int) -> list[Receipt]: score_str=raw_receipt["score_str"], ) + case ReceiptType.JobStillRunningReceipt: + receipt_payload = JobStillRunningReceiptPayload( + job_uuid=raw_receipt["job_uuid"], + miner_hotkey=raw_receipt["miner_hotkey"], + validator_hotkey=raw_receipt["validator_hotkey"], + timestamp=datetime.datetime.fromisoformat(raw_receipt["timestamp"]), + ) + receipt = Receipt( payload=receipt_payload, validator_signature=raw_receipt["validator_signature"], From d566d8fbf78f39339020ea7ddd2c654cf544a256 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Fri, 18 Oct 2024 01:42:51 +0600 Subject: [PATCH 06/40] Move receipt payloads from validator_requests.py to schemas.py --- .../compute_horde/miner_client/organic.py | 3 +- .../mv_protocol/validator_requests.py | 60 +++--------------- .../compute_horde/receipts/models.py | 4 +- .../compute_horde/receipts/schemas.py | 61 +++++++++++++++++-- .../compute_horde/receipts/transfer.py | 5 +- compute_horde/tests/conftest.py | 4 +- compute_horde/tests/test_receipts.py | 5 +- 7 files changed, 73 insertions(+), 69 deletions(-) diff --git a/compute_horde/compute_horde/miner_client/organic.py b/compute_horde/compute_horde/miner_client/organic.py index 2f410f9b3..a0855dd4c 100644 --- a/compute_horde/compute_horde/miner_client/organic.py +++ b/compute_horde/compute_horde/miner_client/organic.py @@ -34,14 +34,13 @@ ) from compute_horde.mv_protocol.validator_requests import ( AuthenticationPayload, - JobFinishedReceiptPayload, - JobStartedReceiptPayload, V0AuthenticateRequest, V0InitialJobRequest, V0JobFinishedReceiptRequest, V0JobRequest, V0JobStartedReceiptRequest, ) +from compute_horde.receipts.schemas import JobFinishedReceiptPayload, JobStartedReceiptPayload from compute_horde.transport import AbstractTransport, TransportConnectionError, WSTransport from compute_horde.utils import MachineSpecs, Timer diff --git a/compute_horde/compute_horde/mv_protocol/validator_requests.py b/compute_horde/compute_horde/mv_protocol/validator_requests.py index ce6344236..d9c2c48d1 100644 --- a/compute_horde/compute_horde/mv_protocol/validator_requests.py +++ b/compute_horde/compute_horde/mv_protocol/validator_requests.py @@ -1,18 +1,22 @@ -import datetime import enum import json import re -from typing import Annotated, Self +from typing import Self import pydantic -from pydantic import field_serializer, model_validator +from pydantic import model_validator from ..base.docker import DockerRunOptionsPreset from ..base.output_upload import OutputUpload # noqa from ..base.volume import Volume, VolumeType from ..base_requests import BaseRequest, JobMixin from ..executor_class import ExecutorClass -from ..utils import MachineSpecs, _json_dumps_default, empty_string_none +from ..receipts.schemas import ( + JobFinishedReceiptPayload, + JobStartedReceiptPayload, + JobStillRunningReceiptPayload, +) +from ..utils import MachineSpecs SAFE_DOMAIN_REGEX = re.compile(r".*") @@ -78,39 +82,6 @@ class GenericError(BaseValidatorRequest): details: str | None = None -class ReceiptPayload(pydantic.BaseModel): - job_uuid: str - miner_hotkey: str - validator_hotkey: str - timestamp: datetime.datetime # when the receipt was generated - - @field_serializer("timestamp") - def serialize_timestamp(self, dt: datetime.datetime, _info): - return dt.isoformat() - - def blob_for_signing(self): - # pydantic v2 does not support sort_keys anymore. - return json.dumps(self.model_dump(), sort_keys=True, default=_json_dumps_default) - - -class JobFinishedReceiptPayload(ReceiptPayload): - time_started: datetime.datetime - time_took_us: int # micro-seconds - score_str: str - - @property - def time_took(self): - return datetime.timedelta(microseconds=self.time_took_us) - - @property - def score(self): - return float(self.score_str) - - @field_serializer("time_started") - def serialize_dt(self, dt: datetime.datetime, _info): - return dt.isoformat() - - class V0JobFinishedReceiptRequest(BaseValidatorRequest): message_type: RequestType = RequestType.V0JobFinishedReceiptRequest payload: JobFinishedReceiptPayload @@ -120,17 +91,6 @@ def blob_for_signing(self): return self.payload.blob_for_signing() -class JobStartedReceiptPayload(ReceiptPayload): - executor_class: ExecutorClass - time_accepted: Annotated[datetime.datetime | None, empty_string_none] - max_timeout: int # seconds - ttl: Annotated[int | None, empty_string_none] = None # seconds - - @field_serializer("time_accepted", when_used="unless-none") - def serialize_dt(self, dt: datetime.datetime, _info): - return dt.isoformat() - - class V0JobStartedReceiptRequest(BaseValidatorRequest): message_type: RequestType = RequestType.V0JobStartedReceiptRequest payload: JobStartedReceiptPayload @@ -158,10 +118,6 @@ def validate_volume_or_volume_type(self) -> Self: return self -class JobStillRunningReceiptPayload(ReceiptPayload): - pass - - class V0JobStillRunningReceiptRequest(BaseValidatorRequest): message_type: RequestType = RequestType.V0JobStillRunningReceiptRequest payload: JobStillRunningReceiptPayload diff --git a/compute_horde/compute_horde/receipts/models.py b/compute_horde/compute_horde/receipts/models.py index b9166034a..642f7f37b 100644 --- a/compute_horde/compute_horde/receipts/models.py +++ b/compute_horde/compute_horde/receipts/models.py @@ -3,11 +3,11 @@ from django.db import models from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS, ExecutorClass -from compute_horde.mv_protocol.validator_requests import ( +from compute_horde.receipts.schemas import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, + Receipt, ) -from compute_horde.receipts.schemas import Receipt class ReceiptNotSigned(Exception): diff --git a/compute_horde/compute_horde/receipts/schemas.py b/compute_horde/compute_horde/receipts/schemas.py index af74ea5df..627499b6f 100644 --- a/compute_horde/compute_horde/receipts/schemas.py +++ b/compute_horde/compute_horde/receipts/schemas.py @@ -1,22 +1,71 @@ +import datetime import enum +import json import logging +from typing import Annotated import bittensor import pydantic +from pydantic import field_serializer -from compute_horde.mv_protocol.validator_requests import ( - JobFinishedReceiptPayload, - JobStartedReceiptPayload, - JobStillRunningReceiptPayload, -) +from compute_horde.executor_class import ExecutorClass +from compute_horde.utils import _json_dumps_default, empty_string_none logger = logging.getLogger(__name__) class ReceiptType(enum.Enum): JobStartedReceipt = "JobStartedReceipt" - JobFinishedReceipt = "JobFinishedReceipt" JobStillRunningReceipt = "JobStillRunningReceipt" + JobFinishedReceipt = "JobFinishedReceipt" + + +class ReceiptPayload(pydantic.BaseModel): + job_uuid: str + miner_hotkey: str + validator_hotkey: str + timestamp: datetime.datetime # when the receipt was generated + + @field_serializer("timestamp") + def serialize_timestamp(self, dt: datetime.datetime, _info): + return dt.isoformat() + + def blob_for_signing(self): + # pydantic v2 does not support sort_keys anymore. + return json.dumps(self.model_dump(), sort_keys=True, default=_json_dumps_default) + + +class JobStartedReceiptPayload(ReceiptPayload): + executor_class: ExecutorClass + time_accepted: Annotated[datetime.datetime | None, empty_string_none] + max_timeout: int # seconds + ttl: Annotated[int | None, empty_string_none] = None # seconds + + @field_serializer("time_accepted", when_used="unless-none") + def serialize_dt(self, dt: datetime.datetime, _info): + return dt.isoformat() + + +class JobStillRunningReceiptPayload(ReceiptPayload): + pass + + +class JobFinishedReceiptPayload(ReceiptPayload): + time_started: datetime.datetime + time_took_us: int # micro-seconds + score_str: str + + @property + def time_took(self): + return datetime.timedelta(microseconds=self.time_took_us) + + @property + def score(self): + return float(self.score_str) + + @field_serializer("time_started") + def serialize_dt(self, dt: datetime.datetime, _info): + return dt.isoformat() class Receipt(pydantic.BaseModel): diff --git a/compute_horde/compute_horde/receipts/transfer.py b/compute_horde/compute_horde/receipts/transfer.py index 58013eec1..a2d3df727 100644 --- a/compute_horde/compute_horde/receipts/transfer.py +++ b/compute_horde/compute_horde/receipts/transfer.py @@ -10,12 +10,13 @@ import requests from compute_horde.executor_class import ExecutorClass -from compute_horde.mv_protocol.validator_requests import ( +from compute_horde.receipts.schemas import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, JobStillRunningReceiptPayload, + Receipt, + ReceiptType, ) -from compute_horde.receipts.schemas import Receipt, ReceiptType logger = logging.getLogger(__name__) diff --git a/compute_horde/tests/conftest.py b/compute_horde/tests/conftest.py index bde902a63..cc4fd69b7 100644 --- a/compute_horde/tests/conftest.py +++ b/compute_horde/tests/conftest.py @@ -5,11 +5,11 @@ from bittensor import Keypair from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS -from compute_horde.mv_protocol.validator_requests import ( +from compute_horde.receipts.schemas import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, + Receipt, ) -from compute_horde.receipts.schemas import Receipt @pytest.fixture diff --git a/compute_horde/tests/test_receipts.py b/compute_horde/tests/test_receipts.py index 545f16a47..ef412e4fd 100644 --- a/compute_horde/tests/test_receipts.py +++ b/compute_horde/tests/test_receipts.py @@ -3,11 +3,10 @@ import pytest -from compute_horde.mv_protocol.validator_requests import ( +from compute_horde.receipts.schemas import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, -) -from compute_horde.receipts.schemas import ( + JobStillRunningReceiptPayload, Receipt, ReceiptType, ) From 28c35db88d4b55d92124052854b92da579c7b664 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Fri, 18 Oct 2024 01:53:35 +0600 Subject: [PATCH 07/40] Add receipt_type discriminator to receipt payloads --- compute_horde/compute_horde/receipts/schemas.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/compute_horde/compute_horde/receipts/schemas.py b/compute_horde/compute_horde/receipts/schemas.py index 627499b6f..a002bc6ca 100644 --- a/compute_horde/compute_horde/receipts/schemas.py +++ b/compute_horde/compute_horde/receipts/schemas.py @@ -6,7 +6,7 @@ import bittensor import pydantic -from pydantic import field_serializer +from pydantic import Field, field_serializer from compute_horde.executor_class import ExecutorClass from compute_horde.utils import _json_dumps_default, empty_string_none @@ -21,6 +21,7 @@ class ReceiptType(enum.Enum): class ReceiptPayload(pydantic.BaseModel): + receipt_type: ReceiptType job_uuid: str miner_hotkey: str validator_hotkey: str @@ -36,6 +37,7 @@ def blob_for_signing(self): class JobStartedReceiptPayload(ReceiptPayload): + receipt_type: ReceiptType = ReceiptType.JobStartedReceipt executor_class: ExecutorClass time_accepted: Annotated[datetime.datetime | None, empty_string_none] max_timeout: int # seconds @@ -47,10 +49,11 @@ def serialize_dt(self, dt: datetime.datetime, _info): class JobStillRunningReceiptPayload(ReceiptPayload): - pass + receipt_type: ReceiptType = ReceiptType.JobStillRunningReceipt class JobFinishedReceiptPayload(ReceiptPayload): + receipt_type: ReceiptType = ReceiptType.JobFinishedReceipt time_started: datetime.datetime time_took_us: int # micro-seconds score_str: str @@ -68,8 +71,14 @@ def serialize_dt(self, dt: datetime.datetime, _info): return dt.isoformat() +ReceiptPayload = Annotated[ + JobStartedReceiptPayload | JobFinishedReceiptPayload | JobStillRunningReceiptPayload, + Field(discriminator="receipt_type"), +] + + class Receipt(pydantic.BaseModel): - payload: JobStartedReceiptPayload | JobFinishedReceiptPayload | JobStillRunningReceiptPayload + payload: ReceiptPayload validator_signature: str miner_signature: str From bc8d687bb717b59a268e53482c050f52a62bedba Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Fri, 18 Oct 2024 01:59:15 +0600 Subject: [PATCH 08/40] Misc changes in schemas.py --- .../compute_horde/receipts/schemas.py | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/compute_horde/compute_horde/receipts/schemas.py b/compute_horde/compute_horde/receipts/schemas.py index a002bc6ca..7b91deaa6 100644 --- a/compute_horde/compute_horde/receipts/schemas.py +++ b/compute_horde/compute_horde/receipts/schemas.py @@ -1,18 +1,14 @@ import datetime import enum import json -import logging from typing import Annotated import bittensor -import pydantic -from pydantic import Field, field_serializer +from pydantic import Field, field_serializer, BaseModel from compute_horde.executor_class import ExecutorClass from compute_horde.utils import _json_dumps_default, empty_string_none -logger = logging.getLogger(__name__) - class ReceiptType(enum.Enum): JobStartedReceipt = "JobStartedReceipt" @@ -20,7 +16,7 @@ class ReceiptType(enum.Enum): JobFinishedReceipt = "JobFinishedReceipt" -class ReceiptPayload(pydantic.BaseModel): +class BaseReceiptPayload(BaseModel): receipt_type: ReceiptType job_uuid: str miner_hotkey: str @@ -36,7 +32,7 @@ def blob_for_signing(self): return json.dumps(self.model_dump(), sort_keys=True, default=_json_dumps_default) -class JobStartedReceiptPayload(ReceiptPayload): +class JobStartedReceiptPayload(BaseReceiptPayload): receipt_type: ReceiptType = ReceiptType.JobStartedReceipt executor_class: ExecutorClass time_accepted: Annotated[datetime.datetime | None, empty_string_none] @@ -44,15 +40,15 @@ class JobStartedReceiptPayload(ReceiptPayload): ttl: Annotated[int | None, empty_string_none] = None # seconds @field_serializer("time_accepted", when_used="unless-none") - def serialize_dt(self, dt: datetime.datetime, _info): + def serialize_time_accepted(self, dt: datetime.datetime, _info): return dt.isoformat() -class JobStillRunningReceiptPayload(ReceiptPayload): +class JobStillRunningReceiptPayload(BaseReceiptPayload): receipt_type: ReceiptType = ReceiptType.JobStillRunningReceipt -class JobFinishedReceiptPayload(ReceiptPayload): +class JobFinishedReceiptPayload(BaseReceiptPayload): receipt_type: ReceiptType = ReceiptType.JobFinishedReceipt time_started: datetime.datetime time_took_us: int # micro-seconds @@ -67,7 +63,7 @@ def score(self): return float(self.score_str) @field_serializer("time_started") - def serialize_dt(self, dt: datetime.datetime, _info): + def serialize_time_started(self, dt: datetime.datetime, _info): return dt.isoformat() @@ -77,7 +73,7 @@ def serialize_dt(self, dt: datetime.datetime, _info): ] -class Receipt(pydantic.BaseModel): +class Receipt(BaseModel): payload: ReceiptPayload validator_signature: str miner_signature: str From 17a2e1811fbd7b8f4596ecf490780b8b0f29ea66 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Fri, 18 Oct 2024 02:05:30 +0600 Subject: [PATCH 09/40] Use pydantic serialization of datetime objects in receipts --- .../compute_horde/receipts/schemas.py | 18 +++--------------- compute_horde/compute_horde/utils.py | 7 ------- 2 files changed, 3 insertions(+), 22 deletions(-) diff --git a/compute_horde/compute_horde/receipts/schemas.py b/compute_horde/compute_horde/receipts/schemas.py index 7b91deaa6..775e980ea 100644 --- a/compute_horde/compute_horde/receipts/schemas.py +++ b/compute_horde/compute_horde/receipts/schemas.py @@ -4,10 +4,10 @@ from typing import Annotated import bittensor -from pydantic import Field, field_serializer, BaseModel +from pydantic import Field, BaseModel from compute_horde.executor_class import ExecutorClass -from compute_horde.utils import _json_dumps_default, empty_string_none +from compute_horde.utils import empty_string_none class ReceiptType(enum.Enum): @@ -23,13 +23,9 @@ class BaseReceiptPayload(BaseModel): validator_hotkey: str timestamp: datetime.datetime # when the receipt was generated - @field_serializer("timestamp") - def serialize_timestamp(self, dt: datetime.datetime, _info): - return dt.isoformat() - def blob_for_signing(self): # pydantic v2 does not support sort_keys anymore. - return json.dumps(self.model_dump(), sort_keys=True, default=_json_dumps_default) + return json.dumps(self.model_dump(mode="json"), sort_keys=True) class JobStartedReceiptPayload(BaseReceiptPayload): @@ -39,10 +35,6 @@ class JobStartedReceiptPayload(BaseReceiptPayload): max_timeout: int # seconds ttl: Annotated[int | None, empty_string_none] = None # seconds - @field_serializer("time_accepted", when_used="unless-none") - def serialize_time_accepted(self, dt: datetime.datetime, _info): - return dt.isoformat() - class JobStillRunningReceiptPayload(BaseReceiptPayload): receipt_type: ReceiptType = ReceiptType.JobStillRunningReceipt @@ -62,10 +54,6 @@ def time_took(self): def score(self): return float(self.score_str) - @field_serializer("time_started") - def serialize_time_started(self, dt: datetime.datetime, _info): - return dt.isoformat() - ReceiptPayload = Annotated[ JobStartedReceiptPayload | JobFinishedReceiptPayload | JobStillRunningReceiptPayload, diff --git a/compute_horde/compute_horde/utils.py b/compute_horde/compute_horde/utils.py index 60d6d71c8..1be8f2a8d 100644 --- a/compute_horde/compute_horde/utils.py +++ b/compute_horde/compute_horde/utils.py @@ -53,13 +53,6 @@ def get_validators(netuid=12, network="finney", block: int | None = None) -> lis return neurons[:VALIDATORS_LIMIT] -def _json_dumps_default(obj): - if isinstance(obj, datetime.datetime): - return obj.isoformat() - - raise TypeError - - class Timer: def __init__(self, timeout=None): self.start_time = datetime.datetime.now() From 8edf43dc22e4d5ef7f55bd07ae0012f4c8c05921 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Fri, 18 Oct 2024 02:13:36 +0600 Subject: [PATCH 10/40] Use JobStartedReceiptPayload only in V0InitialJobRequest Also remove V0JobStartedReceiptRequest --- .../mv_protocol/validator_requests.py | 14 ++------------ compute_horde/compute_horde/receipts/schemas.py | 4 +--- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/compute_horde/compute_horde/mv_protocol/validator_requests.py b/compute_horde/compute_horde/mv_protocol/validator_requests.py index d9c2c48d1..5ed2e0a50 100644 --- a/compute_horde/compute_horde/mv_protocol/validator_requests.py +++ b/compute_horde/compute_horde/mv_protocol/validator_requests.py @@ -91,15 +91,6 @@ def blob_for_signing(self): return self.payload.blob_for_signing() -class V0JobStartedReceiptRequest(BaseValidatorRequest): - message_type: RequestType = RequestType.V0JobStartedReceiptRequest - payload: JobStartedReceiptPayload - signature: str - - def blob_for_signing(self): - return self.payload.blob_for_signing() - - class V0InitialJobRequest(BaseValidatorRequest, JobMixin): message_type: RequestType = RequestType.V0InitialJobRequest executor_class: ExecutorClass | None = None @@ -107,9 +98,8 @@ class V0InitialJobRequest(BaseValidatorRequest, JobMixin): timeout_seconds: int | None = None volume: Volume | None = None volume_type: VolumeType | None = None - - job_started_receipt_payload: JobStartedReceiptPayload | None = None - job_started_receipt_signature: str | None = None + job_started_receipt_payload: JobStartedReceiptPayload + job_started_receipt_signature: str @model_validator(mode="after") def validate_volume_or_volume_type(self) -> Self: diff --git a/compute_horde/compute_horde/receipts/schemas.py b/compute_horde/compute_horde/receipts/schemas.py index 775e980ea..be69f4017 100644 --- a/compute_horde/compute_horde/receipts/schemas.py +++ b/compute_horde/compute_horde/receipts/schemas.py @@ -7,7 +7,6 @@ from pydantic import Field, BaseModel from compute_horde.executor_class import ExecutorClass -from compute_horde.utils import empty_string_none class ReceiptType(enum.Enum): @@ -31,9 +30,8 @@ def blob_for_signing(self): class JobStartedReceiptPayload(BaseReceiptPayload): receipt_type: ReceiptType = ReceiptType.JobStartedReceipt executor_class: ExecutorClass - time_accepted: Annotated[datetime.datetime | None, empty_string_none] max_timeout: int # seconds - ttl: Annotated[int | None, empty_string_none] = None # seconds + ttl: int class JobStillRunningReceiptPayload(BaseReceiptPayload): From 707dc7fd6eabfed4cd198dc2742265220f726066 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Fri, 18 Oct 2024 03:01:32 +0600 Subject: [PATCH 11/40] Rename JobStillRunningReceipt -> JobAcceptedReceipt --- .../compute_horde/mv_protocol/validator_requests.py | 11 +++++------ compute_horde/compute_horde/receipts/schemas.py | 12 +++++++----- compute_horde/compute_horde/receipts/transfer.py | 10 ++++------ compute_horde/tests/test_receipts.py | 1 - 4 files changed, 16 insertions(+), 18 deletions(-) diff --git a/compute_horde/compute_horde/mv_protocol/validator_requests.py b/compute_horde/compute_horde/mv_protocol/validator_requests.py index 5ed2e0a50..547dc5412 100644 --- a/compute_horde/compute_horde/mv_protocol/validator_requests.py +++ b/compute_horde/compute_horde/mv_protocol/validator_requests.py @@ -12,9 +12,9 @@ from ..base_requests import BaseRequest, JobMixin from ..executor_class import ExecutorClass from ..receipts.schemas import ( + JobAcceptedReceiptPayload, JobFinishedReceiptPayload, JobStartedReceiptPayload, - JobStillRunningReceiptPayload, ) from ..utils import MachineSpecs @@ -26,9 +26,8 @@ class RequestType(enum.Enum): V0InitialJobRequest = "V0InitialJobRequest" V0MachineSpecsRequest = "V0MachineSpecsRequest" V0JobRequest = "V0JobRequest" + V0JobAcceptedReceiptRequest = "V0JobAcceptedReceiptRequest" V0JobFinishedReceiptRequest = "V0JobFinishedReceiptRequest" - V0JobStartedReceiptRequest = "V0JobStartedReceiptRequest" - V0JobStillRunningReceiptRequest = "V0JobStillRunningReceiptRequest" GenericError = "GenericError" @@ -108,9 +107,9 @@ def validate_volume_or_volume_type(self) -> Self: return self -class V0JobStillRunningReceiptRequest(BaseValidatorRequest): - message_type: RequestType = RequestType.V0JobStillRunningReceiptRequest - payload: JobStillRunningReceiptPayload +class V0JobAcceptedReceiptRequest(BaseValidatorRequest): + message_type: RequestType = RequestType.V0JobAcceptedReceiptRequest + payload: JobAcceptedReceiptPayload signature: str def blob_for_signing(self): diff --git a/compute_horde/compute_horde/receipts/schemas.py b/compute_horde/compute_horde/receipts/schemas.py index be69f4017..b01115400 100644 --- a/compute_horde/compute_horde/receipts/schemas.py +++ b/compute_horde/compute_horde/receipts/schemas.py @@ -4,14 +4,14 @@ from typing import Annotated import bittensor -from pydantic import Field, BaseModel +from pydantic import BaseModel, Field from compute_horde.executor_class import ExecutorClass class ReceiptType(enum.Enum): JobStartedReceipt = "JobStartedReceipt" - JobStillRunningReceipt = "JobStillRunningReceipt" + JobAcceptedReceipt = "JobAcceptedReceipt" JobFinishedReceipt = "JobFinishedReceipt" @@ -34,8 +34,10 @@ class JobStartedReceiptPayload(BaseReceiptPayload): ttl: int -class JobStillRunningReceiptPayload(BaseReceiptPayload): - receipt_type: ReceiptType = ReceiptType.JobStillRunningReceipt +class JobAcceptedReceiptPayload(BaseReceiptPayload): + receipt_type: ReceiptType = ReceiptType.JobAcceptedReceipt + time_accepted: datetime.datetime + ttl: int class JobFinishedReceiptPayload(BaseReceiptPayload): @@ -54,7 +56,7 @@ def score(self): ReceiptPayload = Annotated[ - JobStartedReceiptPayload | JobFinishedReceiptPayload | JobStillRunningReceiptPayload, + JobStartedReceiptPayload | JobAcceptedReceiptPayload | JobFinishedReceiptPayload, Field(discriminator="receipt_type"), ] diff --git a/compute_horde/compute_horde/receipts/transfer.py b/compute_horde/compute_horde/receipts/transfer.py index a2d3df727..6f8671c61 100644 --- a/compute_horde/compute_horde/receipts/transfer.py +++ b/compute_horde/compute_horde/receipts/transfer.py @@ -11,9 +11,9 @@ from compute_horde.executor_class import ExecutorClass from compute_horde.receipts.schemas import ( + JobAcceptedReceiptPayload, JobFinishedReceiptPayload, JobStartedReceiptPayload, - JobStillRunningReceiptPayload, Receipt, ReceiptType, ) @@ -46,9 +46,7 @@ def get_miner_receipts(hotkey: str, ip: str, port: int) -> list[Receipt]: try: receipt_type = ReceiptType(raw_receipt["type"]) receipt_payload: ( - JobStartedReceiptPayload - | JobFinishedReceiptPayload - | JobStillRunningReceiptPayload + JobStartedReceiptPayload | JobFinishedReceiptPayload | JobAcceptedReceiptPayload ) match receipt_type: @@ -76,8 +74,8 @@ def get_miner_receipts(hotkey: str, ip: str, port: int) -> list[Receipt]: score_str=raw_receipt["score_str"], ) - case ReceiptType.JobStillRunningReceipt: - receipt_payload = JobStillRunningReceiptPayload( + case ReceiptType.JobAcceptedReceipt: + receipt_payload = JobAcceptedReceiptPayload( job_uuid=raw_receipt["job_uuid"], miner_hotkey=raw_receipt["miner_hotkey"], validator_hotkey=raw_receipt["validator_hotkey"], diff --git a/compute_horde/tests/test_receipts.py b/compute_horde/tests/test_receipts.py index ef412e4fd..baa8619b5 100644 --- a/compute_horde/tests/test_receipts.py +++ b/compute_horde/tests/test_receipts.py @@ -6,7 +6,6 @@ from compute_horde.receipts.schemas import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, - JobStillRunningReceiptPayload, Receipt, ReceiptType, ) From 8630ce3d774035d0cd4e106a37683ae44abaa530 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Fri, 18 Oct 2024 03:03:12 +0600 Subject: [PATCH 12/40] Undo move of V0InitialJobRequest --- .../mv_protocol/validator_requests.py | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/compute_horde/compute_horde/mv_protocol/validator_requests.py b/compute_horde/compute_horde/mv_protocol/validator_requests.py index 547dc5412..326422c15 100644 --- a/compute_horde/compute_horde/mv_protocol/validator_requests.py +++ b/compute_horde/compute_horde/mv_protocol/validator_requests.py @@ -54,6 +54,23 @@ def blob_for_signing(self): return self.payload.blob_for_signing() +class V0InitialJobRequest(BaseValidatorRequest, JobMixin): + message_type: RequestType = RequestType.V0InitialJobRequest + executor_class: ExecutorClass | None = None + base_docker_image_name: str | None = None + timeout_seconds: int | None = None + volume: Volume | None = None + volume_type: VolumeType | None = None + job_started_receipt_payload: JobStartedReceiptPayload + job_started_receipt_signature: str + + @model_validator(mode="after") + def validate_volume_or_volume_type(self) -> Self: + if bool(self.volume) and bool(self.volume_type): + raise ValueError("Expected either `volume` or `volume_type`, got both") + return self + + class V0JobRequest(BaseValidatorRequest, JobMixin): message_type: RequestType = RequestType.V0JobRequest executor_class: ExecutorClass | None = None @@ -90,23 +107,6 @@ def blob_for_signing(self): return self.payload.blob_for_signing() -class V0InitialJobRequest(BaseValidatorRequest, JobMixin): - message_type: RequestType = RequestType.V0InitialJobRequest - executor_class: ExecutorClass | None = None - base_docker_image_name: str | None = None - timeout_seconds: int | None = None - volume: Volume | None = None - volume_type: VolumeType | None = None - job_started_receipt_payload: JobStartedReceiptPayload - job_started_receipt_signature: str - - @model_validator(mode="after") - def validate_volume_or_volume_type(self) -> Self: - if bool(self.volume) and bool(self.volume_type): - raise ValueError("Expected either `volume` or `volume_type`, got both") - return self - - class V0JobAcceptedReceiptRequest(BaseValidatorRequest): message_type: RequestType = RequestType.V0JobAcceptedReceiptRequest payload: JobAcceptedReceiptPayload From 53c08873de3fd5c9a94d97e64cdadfda67d68163 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Fri, 18 Oct 2024 03:15:03 +0600 Subject: [PATCH 13/40] Fix receipt payload discriminator --- compute_horde/compute_horde/receipts/schemas.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/compute_horde/compute_horde/receipts/schemas.py b/compute_horde/compute_horde/receipts/schemas.py index b01115400..6d56930d8 100644 --- a/compute_horde/compute_horde/receipts/schemas.py +++ b/compute_horde/compute_horde/receipts/schemas.py @@ -1,7 +1,7 @@ import datetime import enum import json -from typing import Annotated +from typing import Annotated, Literal import bittensor from pydantic import BaseModel, Field @@ -16,7 +16,6 @@ class ReceiptType(enum.Enum): class BaseReceiptPayload(BaseModel): - receipt_type: ReceiptType job_uuid: str miner_hotkey: str validator_hotkey: str @@ -28,20 +27,20 @@ def blob_for_signing(self): class JobStartedReceiptPayload(BaseReceiptPayload): - receipt_type: ReceiptType = ReceiptType.JobStartedReceipt + receipt_type: Literal[ReceiptType.JobStartedReceipt] = ReceiptType.JobStartedReceipt executor_class: ExecutorClass max_timeout: int # seconds ttl: int class JobAcceptedReceiptPayload(BaseReceiptPayload): - receipt_type: ReceiptType = ReceiptType.JobAcceptedReceipt + receipt_type: Literal[ReceiptType.JobAcceptedReceipt] = ReceiptType.JobAcceptedReceipt time_accepted: datetime.datetime ttl: int class JobFinishedReceiptPayload(BaseReceiptPayload): - receipt_type: ReceiptType = ReceiptType.JobFinishedReceipt + receipt_type: Literal[ReceiptType.JobFinishedReceipt] = ReceiptType.JobFinishedReceipt time_started: datetime.datetime time_took_us: int # micro-seconds score_str: str From 7761bcc8714dd14c555837357379394cf32b816b Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Fri, 18 Oct 2024 04:14:16 +0600 Subject: [PATCH 14/40] Add receipt model fields and fix mypy errors of lib --- .../compute_horde/miner_client/organic.py | 73 +++++++++++++------ .../compute_horde/receipts/models.py | 33 ++++++++- .../compute_horde/receipts/transfer.py | 20 +++-- 3 files changed, 92 insertions(+), 34 deletions(-) diff --git a/compute_horde/compute_horde/miner_client/organic.py b/compute_horde/compute_horde/miner_client/organic.py index a0855dd4c..f66b4d7bb 100644 --- a/compute_horde/compute_horde/miner_client/organic.py +++ b/compute_horde/compute_horde/miner_client/organic.py @@ -13,7 +13,7 @@ from compute_horde.base.output_upload import OutputUpload from compute_horde.base.volume import Volume from compute_horde.base_requests import BaseRequest -from compute_horde.executor_class import ExecutorClass +from compute_horde.executor_class import EXECUTOR_CLASS, ExecutorClass from compute_horde.miner_client.base import ( AbstractMinerClient, ErrorCallback, @@ -36,16 +36,22 @@ AuthenticationPayload, V0AuthenticateRequest, V0InitialJobRequest, + V0JobAcceptedReceiptRequest, V0JobFinishedReceiptRequest, V0JobRequest, - V0JobStartedReceiptRequest, ) -from compute_horde.receipts.schemas import JobFinishedReceiptPayload, JobStartedReceiptPayload +from compute_horde.receipts.schemas import ( + JobAcceptedReceiptPayload, + JobFinishedReceiptPayload, + JobStartedReceiptPayload, +) from compute_horde.transport import AbstractTransport, TransportConnectionError, WSTransport from compute_horde.utils import MachineSpecs, Timer logger = logging.getLogger(__name__) +JOB_STARTED_RECEIPT_MIN_TTL = 30 + class OrganicMinerClient(AbstractMinerClient): """ @@ -216,39 +222,54 @@ def generate_authentication_message(self) -> V0AuthenticateRequest: def generate_job_started_receipt_message( self, executor_class: ExecutorClass, - accepted_timestamp: float, max_timeout: int, - ) -> V0JobStartedReceiptRequest: - time_accepted = datetime.datetime.fromtimestamp(accepted_timestamp, datetime.UTC) + ttl: int, + ) -> tuple[JobStartedReceiptPayload, str]: receipt_payload = JobStartedReceiptPayload( job_uuid=self.job_uuid, miner_hotkey=self.miner_hotkey, validator_hotkey=self.my_hotkey, + timestamp=datetime.datetime.now(datetime.UTC), executor_class=executor_class, - time_accepted=time_accepted, max_timeout=max_timeout, + ttl=ttl, + ) + signature = f"0x{self.my_keypair.sign(receipt_payload.blob_for_signing()).hex()}" + return receipt_payload, signature + + def generate_job_accepted_receipt_message( + self, + accepted_timestamp: float, + ttl: int, + ) -> V0JobAcceptedReceiptRequest: + time_accepted = datetime.datetime.fromtimestamp(accepted_timestamp, datetime.UTC) + receipt_payload = JobAcceptedReceiptPayload( + job_uuid=self.job_uuid, + miner_hotkey=self.miner_hotkey, + validator_hotkey=self.my_hotkey, + timestamp=datetime.datetime.now(datetime.UTC), + time_accepted=time_accepted, + ttl=ttl, ) - return V0JobStartedReceiptRequest( + return V0JobAcceptedReceiptRequest( payload=receipt_payload, signature=f"0x{self.my_keypair.sign(receipt_payload.blob_for_signing()).hex()}", ) - async def send_job_started_receipt_message( + async def send_job_accepted_receipt_message( self, - executor_class: ExecutorClass, accepted_timestamp: float, - max_timeout: int, + ttl: int, ) -> None: try: - receipt_message = self.generate_job_started_receipt_message( - executor_class, + receipt_message = self.generate_job_accepted_receipt_message( accepted_timestamp, - max_timeout, + ttl, ) await self.send_model(receipt_message) - logger.debug(f"Sent job started receipt for {self.job_uuid}") + logger.debug(f"Sent job accepted receipt for {self.job_uuid}") except Exception as e: - comment = f"Failed to send job started receipt to miner {self.miner_name} for job {self.job_uuid}: {e}" + comment = f"Failed to send job accepted receipt to miner {self.miner_name} for job {self.job_uuid}: {e}" logger.warning(comment) await self.notify_receipt_failure(comment) @@ -263,6 +284,7 @@ def generate_job_finished_receipt_message( job_uuid=self.job_uuid, miner_hotkey=self.miner_hotkey, validator_hotkey=self.my_hotkey, + timestamp=datetime.datetime.now(datetime.UTC), time_started=time_started, time_took_us=int(time_took_seconds * 1_000_000), score_str=f"{score:.6f}", @@ -318,9 +340,9 @@ def __init__(self, reason: FailureReason, received: BaseRequest | None = None): self.received = received def __str__(self): - s = f"Organic job failed, received: {self.received_str()}" + s = f"Organic job failed, {self.reason=}" if self.received: - s += f", {self.received=}" + s += f", received: {self.received_str()}" return s def __repr__(self): @@ -372,6 +394,14 @@ async def run_organic_job( job_timer = Timer(timeout=job_details.total_job_timeout) + receipt_payload, receipt_signature = client.generate_job_started_receipt_message( + executor_class=job_details.executor_class, + max_timeout=int(job_timer.time_left()), + ttl=max( + JOB_STARTED_RECEIPT_MIN_TTL, + EXECUTOR_CLASS[job_details.executor_class].spin_up_time or 0, + ), + ) await client.send_model( V0InitialJobRequest( job_uuid=job_details.job_uuid, @@ -379,6 +409,8 @@ async def run_organic_job( base_docker_image_name=job_details.docker_image, timeout_seconds=job_details.total_job_timeout, volume_type=job_details.volume.volume_type if job_details.volume else None, + job_started_receipt_payload=receipt_payload, + job_started_receipt_signature=receipt_signature, ), ) @@ -406,10 +438,9 @@ async def run_organic_job( await client.notify_executor_ready(executor_readiness_response) - await client.send_job_started_receipt_message( - executor_class=job_details.executor_class, + await client.send_job_accepted_receipt_message( accepted_timestamp=time.time(), - max_timeout=int(job_timer.time_left()), + ttl=int(job_timer.time_left()), ) await client.send_model( diff --git a/compute_horde/compute_horde/receipts/models.py b/compute_horde/compute_horde/receipts/models.py index 642f7f37b..9298f408b 100644 --- a/compute_horde/compute_horde/receipts/models.py +++ b/compute_horde/compute_horde/receipts/models.py @@ -4,6 +4,7 @@ from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS, ExecutorClass from compute_horde.receipts.schemas import ( + JobAcceptedReceiptPayload, JobFinishedReceiptPayload, JobStartedReceiptPayload, Receipt, @@ -20,6 +21,10 @@ class AbstractReceipt(models.Model): miner_hotkey = models.CharField(max_length=256) validator_signature = models.CharField(max_length=256) miner_signature = models.CharField(max_length=256, null=True, blank=True) + timestamp = models.DateTimeField() + + # https://github.com/typeddjango/django-stubs/issues/1684#issuecomment-1706446344 + objects: models.Manager["JobStartedReceipt"] class Meta: abstract = True @@ -33,8 +38,8 @@ def __str__(self): class JobStartedReceipt(AbstractReceipt): executor_class = models.CharField(max_length=255, default=DEFAULT_EXECUTOR_CLASS) - time_accepted = models.DateTimeField() max_timeout = models.IntegerField() + ttl = models.IntegerField() # https://github.com/typeddjango/django-stubs/issues/1684#issuecomment-1706446344 objects: models.Manager["JobStartedReceipt"] @@ -48,9 +53,32 @@ def to_receipt(self) -> Receipt: job_uuid=str(self.job_uuid), miner_hotkey=self.miner_hotkey, validator_hotkey=self.validator_hotkey, + timestamp=self.timestamp, executor_class=ExecutorClass(self.executor_class), - time_accepted=self.time_accepted, max_timeout=self.max_timeout, + ttl=self.ttl, + ), + validator_signature=self.validator_signature, + miner_signature=self.miner_signature, + ) + + +class JobAcceptedReceipt(AbstractReceipt): + time_accepted = models.DateTimeField() + ttl = models.IntegerField() + + def to_receipt(self) -> Receipt: + if self.miner_signature is None: + raise ReceiptNotSigned("Miner signature is required") + + return Receipt( + payload=JobAcceptedReceiptPayload( + job_uuid=str(self.job_uuid), + miner_hotkey=self.miner_hotkey, + validator_hotkey=self.validator_hotkey, + timestamp=self.timestamp, + time_accepted=self.time_accepted, + ttl=self.ttl, ), validator_signature=self.validator_signature, miner_signature=self.miner_signature, @@ -80,6 +108,7 @@ def to_receipt(self) -> Receipt: job_uuid=str(self.job_uuid), miner_hotkey=self.miner_hotkey, validator_hotkey=self.validator_hotkey, + timestamp=self.timestamp, time_started=self.time_started, time_took_us=self.time_took_us, score_str=self.score_str, diff --git a/compute_horde/compute_horde/receipts/transfer.py b/compute_horde/compute_horde/receipts/transfer.py index 6f8671c61..adcc737a4 100644 --- a/compute_horde/compute_horde/receipts/transfer.py +++ b/compute_horde/compute_horde/receipts/transfer.py @@ -1,6 +1,5 @@ import contextlib import csv -import datetime import io import logging import shutil @@ -15,6 +14,7 @@ JobFinishedReceiptPayload, JobStartedReceiptPayload, Receipt, + ReceiptPayload, ReceiptType, ) @@ -45,9 +45,7 @@ def get_miner_receipts(hotkey: str, ip: str, port: int) -> list[Receipt]: for raw_receipt in csv_reader: try: receipt_type = ReceiptType(raw_receipt["type"]) - receipt_payload: ( - JobStartedReceiptPayload | JobFinishedReceiptPayload | JobAcceptedReceiptPayload - ) + receipt_payload: ReceiptPayload match receipt_type: case ReceiptType.JobStartedReceipt: @@ -55,11 +53,10 @@ def get_miner_receipts(hotkey: str, ip: str, port: int) -> list[Receipt]: job_uuid=raw_receipt["job_uuid"], miner_hotkey=raw_receipt["miner_hotkey"], validator_hotkey=raw_receipt["validator_hotkey"], + timestamp=raw_receipt["timestamp"], # type: ignore[arg-type] executor_class=ExecutorClass(raw_receipt["executor_class"]), - time_accepted=datetime.datetime.fromisoformat( - raw_receipt["time_accepted"] - ), max_timeout=int(raw_receipt["max_timeout"]), + ttl=int(raw_receipt["ttl"]), ) case ReceiptType.JobFinishedReceipt: @@ -67,9 +64,8 @@ def get_miner_receipts(hotkey: str, ip: str, port: int) -> list[Receipt]: job_uuid=raw_receipt["job_uuid"], miner_hotkey=raw_receipt["miner_hotkey"], validator_hotkey=raw_receipt["validator_hotkey"], - time_started=datetime.datetime.fromisoformat( - raw_receipt["time_started"] - ), + timestamp=raw_receipt["timestamp"], # type: ignore[arg-type] + time_started=raw_receipt["time_started"], # type: ignore[arg-type] time_took_us=int(raw_receipt["time_took_us"]), score_str=raw_receipt["score_str"], ) @@ -79,7 +75,9 @@ def get_miner_receipts(hotkey: str, ip: str, port: int) -> list[Receipt]: job_uuid=raw_receipt["job_uuid"], miner_hotkey=raw_receipt["miner_hotkey"], validator_hotkey=raw_receipt["validator_hotkey"], - timestamp=datetime.datetime.fromisoformat(raw_receipt["timestamp"]), + timestamp=raw_receipt["timestamp"], # type: ignore[arg-type] + time_accepted=raw_receipt["time_accepted"], # type: ignore[arg-type] + ttl=int(raw_receipt["ttl"]), ) receipt = Receipt( From 81f337f0575d9cad097cbabb4f73c312ae42e841 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Fri, 18 Oct 2024 14:46:40 +0600 Subject: [PATCH 15/40] Send finished receipt when organic job fails --- .../compute_horde/miner_client/organic.py | 103 ++++++++++-------- 1 file changed, 56 insertions(+), 47 deletions(-) diff --git a/compute_horde/compute_horde/miner_client/organic.py b/compute_horde/compute_horde/miner_client/organic.py index f66b4d7bb..32fd456ae 100644 --- a/compute_horde/compute_horde/miner_client/organic.py +++ b/compute_horde/compute_horde/miner_client/organic.py @@ -415,60 +415,69 @@ async def run_organic_job( ) try: - initial_response = await asyncio.wait_for( - client.miner_accepting_or_declining_future, - timeout=min(job_timer.time_left(), wait_timeout), - ) - except TimeoutError as exc: - raise OrganicJobError(FailureReason.INITIAL_RESPONSE_TIMED_OUT) from exc - if isinstance(initial_response, V0DeclineJobRequest): - raise OrganicJobError(FailureReason.JOB_DECLINED, initial_response) + try: + initial_response = await asyncio.wait_for( + client.miner_accepting_or_declining_future, + timeout=min(job_timer.time_left(), wait_timeout), + ) + except TimeoutError as exc: + raise OrganicJobError(FailureReason.INITIAL_RESPONSE_TIMED_OUT) from exc + if isinstance(initial_response, V0DeclineJobRequest): + raise OrganicJobError(FailureReason.JOB_DECLINED, initial_response) - await client.notify_job_accepted(initial_response) + await client.notify_job_accepted(initial_response) - try: - executor_readiness_response = await asyncio.wait_for( - client.executor_ready_or_failed_future, - timeout=min(job_timer.time_left(), wait_timeout), + try: + executor_readiness_response = await asyncio.wait_for( + client.executor_ready_or_failed_future, + timeout=min(job_timer.time_left(), wait_timeout), + ) + except TimeoutError as exc: + raise OrganicJobError(FailureReason.EXECUTOR_READINESS_RESPONSE_TIMED_OUT) from exc + if isinstance(executor_readiness_response, V0ExecutorFailedRequest): + raise OrganicJobError(FailureReason.EXECUTOR_FAILED, executor_readiness_response) + + await client.notify_executor_ready(executor_readiness_response) + + await client.send_job_accepted_receipt_message( + accepted_timestamp=time.time(), + ttl=int(job_timer.time_left()), ) - except TimeoutError as exc: - raise OrganicJobError(FailureReason.EXECUTOR_READINESS_RESPONSE_TIMED_OUT) from exc - if isinstance(executor_readiness_response, V0ExecutorFailedRequest): - raise OrganicJobError(FailureReason.EXECUTOR_FAILED, executor_readiness_response) - - await client.notify_executor_ready(executor_readiness_response) - - await client.send_job_accepted_receipt_message( - accepted_timestamp=time.time(), - ttl=int(job_timer.time_left()), - ) - await client.send_model( - V0JobRequest( - job_uuid=job_details.job_uuid, - executor_class=job_details.executor_class, - docker_image_name=job_details.docker_image, - raw_script=job_details.raw_script, - docker_run_options_preset=job_details.docker_run_options_preset, - docker_run_cmd=job_details.docker_run_cmd, - volume=job_details.volume, - output_upload=job_details.output, + await client.send_model( + V0JobRequest( + job_uuid=job_details.job_uuid, + executor_class=job_details.executor_class, + docker_image_name=job_details.docker_image, + raw_script=job_details.raw_script, + docker_run_options_preset=job_details.docker_run_options_preset, + docker_run_cmd=job_details.docker_run_cmd, + volume=job_details.volume, + output_upload=job_details.output, + ) ) - ) - try: - final_response = await asyncio.wait_for( - client.miner_finished_or_failed_future, - timeout=job_timer.time_left(), - ) - if isinstance(final_response, V0JobFailedRequest): - raise OrganicJobError(FailureReason.JOB_FAILED, final_response) - return final_response.docker_process_stdout, final_response.docker_process_stderr - except TimeoutError as exc: - raise OrganicJobError(FailureReason.FINAL_RESPONSE_TIMED_OUT) from exc - finally: + try: + final_response = await asyncio.wait_for( + client.miner_finished_or_failed_future, + timeout=job_timer.time_left(), + ) + if isinstance(final_response, V0JobFailedRequest): + raise OrganicJobError(FailureReason.JOB_FAILED, final_response) + + await client.send_job_finished_receipt_message( + started_timestamp=job_timer.start_time.timestamp(), + time_took_seconds=job_timer.passed_time(), + score=0, # no score for organic jobs (at least right now) + ) + + return final_response.docker_process_stdout, final_response.docker_process_stderr + except TimeoutError as exc: + raise OrganicJobError(FailureReason.FINAL_RESPONSE_TIMED_OUT) from exc + except Exception: await client.send_job_finished_receipt_message( started_timestamp=job_timer.start_time.timestamp(), time_took_seconds=job_timer.passed_time(), - score=0, # no score for organic jobs (at least right now) + score=0, ) + raise From 7141448fcd9bd8aada30e4b3d1829dd043dc3657 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sat, 19 Oct 2024 00:53:27 +0600 Subject: [PATCH 16/40] Fix surface level errors in miner --- .../miner/liveness_check.py | 15 ++++ .../miner_consumer/validator_interface.py | 81 +++++++++++------ .../miner/receipt_store/local.py | 13 ++- .../src/compute_horde_miner/miner/tasks.py | 88 +++++++++++-------- 4 files changed, 124 insertions(+), 73 deletions(-) diff --git a/miner/app/src/compute_horde_miner/miner/liveness_check.py b/miner/app/src/compute_horde_miner/miner/liveness_check.py index 486d1267d..c400a76a2 100644 --- a/miner/app/src/compute_horde_miner/miner/liveness_check.py +++ b/miner/app/src/compute_horde_miner/miner/liveness_check.py @@ -1,5 +1,6 @@ import asyncio import base64 +import datetime import io import json import os @@ -14,6 +15,7 @@ from compute_horde.base.volume import InlineVolume, VolumeType from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS from compute_horde.mv_protocol import validator_requests +from compute_horde.receipts.schemas import JobStartedReceiptPayload from django.conf import settings from compute_horde_miner.channel_layer.channel_layer import ECRedisChannelLayer @@ -148,12 +150,25 @@ async def drive_executor() -> float: job_uuid = str(uuid.uuid4()) executor_token = f"{job_uuid}-{uuid.uuid4()}" + keypair = settings.BITTENSOR_WALLET().get_hotkey() + receipt_payload = JobStartedReceiptPayload( + job_uuid=job_uuid, + miner_hotkey=keypair.ss58_address, + validator_hotkey=keypair.ss58_address, + timestamp=datetime.datetime.now(datetime.UTC), + executor_class=DEFAULT_EXECUTOR_CLASS, + max_timeout=JOB_TIMEOUT_SECONDS, + ttl=30, + ) + receipt_signature = f"0x{keypair.sign(receipt_payload.blob_for_signing()).hex()}" initial_job_request = validator_requests.V0InitialJobRequest( job_uuid=job_uuid, executor_class=DEFAULT_EXECUTOR_CLASS, base_docker_image_name=JOB_IMAGE_NAME, timeout_seconds=JOB_TIMEOUT_SECONDS, volume_type=VolumeType.inline, + job_started_receipt_payload=receipt_payload, + job_started_receipt_signature=receipt_signature, ) job_request = validator_requests.V0JobRequest( job_uuid=job_uuid, diff --git a/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py b/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py index ee74158cd..5fb98ae82 100644 --- a/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py +++ b/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py @@ -10,7 +10,8 @@ from compute_horde.mv_protocol.validator_requests import ( BaseValidatorRequest, ) -from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.models import JobAcceptedReceipt, JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.schemas import JobStartedReceiptPayload, ReceiptPayload from django.conf import settings from django.utils import timezone @@ -156,32 +157,26 @@ def verify_auth_msg(self, msg: validator_requests.V0AuthenticateRequest) -> tupl return False, "Signature mismatches" - def verify_receipt_msg( - self, - msg: validator_requests.V0JobStartedReceiptRequest - | validator_requests.V0JobFinishedReceiptRequest, - ) -> bool: + def verify_receipt_payload(self, payload: ReceiptPayload, signature: str) -> bool: if settings.IS_LOCAL_MINER: return True - if self.my_hotkey != DONT_CHECK and msg.payload.miner_hotkey != self.my_hotkey: + if self.my_hotkey != DONT_CHECK and payload.miner_hotkey != self.my_hotkey: logger.warning( - f"Miner hotkey mismatch in receipt for job_uuid {msg.payload.job_uuid} ({msg.payload.miner_hotkey!r} != {self.my_hotkey!r})" + f"Miner hotkey mismatch in receipt for job_uuid {payload.job_uuid} ({payload.miner_hotkey!r} != {self.my_hotkey!r})" ) return False - if msg.payload.validator_hotkey != self.validator_key: + if payload.validator_hotkey != self.validator_key: logger.warning( - f"Validator hotkey mismatch in receipt for job_uuid {msg.payload.job_uuid} ({msg.payload.validator_hotkey!r} != {self.validator_key!r})" + f"Validator hotkey mismatch in receipt for job_uuid {payload.job_uuid} ({payload.validator_hotkey!r} != {self.validator_key!r})" ) return False keypair = bittensor.Keypair(ss58_address=self.validator_key) - if keypair.verify(msg.blob_for_signing(), msg.signature): + if keypair.verify(payload.blob_for_signing(), signature): return True - logger.warning( - f"Validator signature mismatch in receipt for job_uuid {msg.payload.job_uuid}" - ) + logger.warning(f"Validator signature mismatch in receipt for job_uuid {payload.job_uuid}") return False async def handle_authentication(self, msg: validator_requests.V0AuthenticateRequest): @@ -299,13 +294,13 @@ async def handle(self, msg: BaseValidatorRequest): await self.handle_job_request(msg) if isinstance( - msg, validator_requests.V0JobStartedReceiptRequest - ) and self.verify_receipt_msg(msg): - await self.handle_job_started_receipt(msg) + msg, validator_requests.V0JobAcceptedReceiptRequest + ) and self.verify_receipt_payload(msg.payload, msg.signature): + await self.handle_job_accepted_receipt(msg) if isinstance( msg, validator_requests.V0JobFinishedReceiptRequest - ) and self.verify_receipt_msg(msg): + ) and self.verify_receipt_payload(msg.payload, msg.signature): await self.handle_job_finished_receipt(msg) async def handle_initial_job_request(self, msg: validator_requests.V0InitialJobRequest): @@ -320,6 +315,11 @@ async def handle_initial_job_request(self, msg: validator_requests.V0InitialJobR miner_requests.V0DeclineJobRequest(job_uuid=msg.job_uuid).model_dump_json() ) return + + await self.handle_job_started_receipt( + msg.job_started_receipt_payload, msg.job_started_receipt_signature + ) + # TODO add rate limiting per validator key here token = f"{msg.job_uuid}-{uuid.uuid4()}" await self.group_add(token) @@ -378,26 +378,52 @@ async def handle_job_request(self, msg: validator_requests.V0JobRequest): job.full_job_details = msg.model_dump() await job.asave() - async def handle_job_started_receipt(self, msg: validator_requests.V0JobStartedReceiptRequest): + async def handle_job_started_receipt(self, payload: JobStartedReceiptPayload, signature: str): logger.info( f"Received job started receipt for" - f" job_uuid={msg.payload.job_uuid} validator_hotkey={msg.payload.validator_hotkey}" - f" max_timeout={msg.payload.max_timeout}" + f" job_uuid={payload.job_uuid} validator_hotkey={payload.validator_hotkey}" + f" max_timeout={payload.max_timeout}" ) if settings.IS_LOCAL_MINER: return + if not self.verify_receipt_payload(payload, signature): + return + await JobStartedReceipt.objects.acreate( - job_uuid=msg.payload.job_uuid, - validator_hotkey=msg.payload.validator_hotkey, - miner_hotkey=msg.payload.miner_hotkey, + job_uuid=payload.job_uuid, + validator_hotkey=payload.validator_hotkey, + miner_hotkey=payload.miner_hotkey, + validator_signature=signature, + miner_signature=get_miner_signature(payload), + timestamp=payload.timestamp, + executor_class=payload.executor_class, + max_timeout=payload.max_timeout, + ttl=payload.ttl, + ) + + async def handle_job_accepted_receipt( + self, msg: validator_requests.V0JobAcceptedReceiptRequest + ): + logger.info( + f"Received job accepted receipt for" + f" job_uuid={msg.payload.job_uuid} validator_hotkey={msg.payload.validator_hotkey}" + ) + + if settings.IS_LOCAL_MINER: + return + + await JobAcceptedReceipt.objects.acreate( validator_signature=msg.signature, miner_signature=get_miner_signature(msg), - executor_class=msg.payload.executor_class, - time_accepted=msg.payload.time_accepted, - max_timeout=msg.payload.max_timeout, + job_uuid=msg.payload.job_uuid, + miner_hotkey=msg.payload.miner_hotkey, + validator_hotkey=msg.payload.validator_hotkey, + timestamp=msg.payload.timestamp, + ttl=msg.payload.ttl, ) + prepare_receipts.delay() async def handle_job_finished_receipt( self, msg: validator_requests.V0JobFinishedReceiptRequest @@ -421,6 +447,7 @@ async def handle_job_finished_receipt( job_uuid=msg.payload.job_uuid, miner_hotkey=msg.payload.miner_hotkey, validator_hotkey=msg.payload.validator_hotkey, + timestamp=msg.payload.timestamp, time_started=msg.payload.time_started, time_took_us=msg.payload.time_took_us, score_str=msg.payload.score_str, diff --git a/miner/app/src/compute_horde_miner/miner/receipt_store/local.py b/miner/app/src/compute_horde_miner/miner/receipt_store/local.py index 2aad9143e..c819ccd2f 100644 --- a/miner/app/src/compute_horde_miner/miner/receipt_store/local.py +++ b/miner/app/src/compute_horde_miner/miner/receipt_store/local.py @@ -4,11 +4,12 @@ import shutil import tempfile -from compute_horde.mv_protocol.validator_requests import ( +from compute_horde.receipts.schemas import ( + JobAcceptedReceiptPayload, JobFinishedReceiptPayload, JobStartedReceiptPayload, + Receipt, ) -from compute_horde.receipts.schemas import Receipt, ReceiptType from django.conf import settings from compute_horde_miner.miner.receipt_store.base import BaseReceiptStore @@ -23,6 +24,7 @@ def store(self, receipts: list[Receipt]) -> None: payload_fields = set() payload_fields |= set(JobStartedReceiptPayload.model_fields.keys()) + payload_fields |= set(JobAcceptedReceiptPayload.model_fields.keys()) payload_fields |= set(JobFinishedReceiptPayload.model_fields.keys()) buf = io.StringIO() @@ -37,14 +39,9 @@ def store(self, receipts: list[Receipt]) -> None: ) csv_writer.writeheader() for receipt in receipts: - match receipt.payload: - case JobStartedReceiptPayload(): - receipt_type = ReceiptType.JobStartedReceipt - case JobFinishedReceiptPayload(): - receipt_type = ReceiptType.JobFinishedReceipt row = ( dict( - type=receipt_type.value, + type=receipt.payload.receipt_type.value, validator_signature=receipt.validator_signature, miner_signature=receipt.miner_signature, ) diff --git a/miner/app/src/compute_horde_miner/miner/tasks.py b/miner/app/src/compute_horde_miner/miner/tasks.py index 8a3d987cc..cc7edf44f 100644 --- a/miner/app/src/compute_horde_miner/miner/tasks.py +++ b/miner/app/src/compute_horde_miner/miner/tasks.py @@ -2,11 +2,12 @@ from celery.utils.log import get_task_logger from compute_horde.dynamic_config import sync_dynamic_config -from compute_horde.mv_protocol.validator_requests import ( +from compute_horde.receipts.models import JobAcceptedReceipt, JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.schemas import ( + JobAcceptedReceiptPayload, JobFinishedReceiptPayload, JobStartedReceiptPayload, ) -from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt from compute_horde.receipts.transfer import get_miner_receipts from compute_horde.utils import get_validators from constance import config @@ -70,27 +71,15 @@ def fetch_validators(): def prepare_receipts(): receipts = [] - job_started_receipts = JobStartedReceipt.objects.order_by("time_accepted").filter( - time_accepted__gt=now() - RECEIPTS_MAX_SERVED_PERIOD - ) - for job_started_receipt in job_started_receipts: - try: - receipts.append(job_started_receipt.to_receipt()) - except Exception as e: - logger.error( - f"Skipping job started receipt for job {job_started_receipt.job_uuid}: {e}" - ) - - job_finished_receipts = JobFinishedReceipt.objects.order_by("time_started").filter( - time_started__gt=now() - RECEIPTS_MAX_SERVED_PERIOD - ) - for job_finished_receipt in job_finished_receipts: - try: - receipts.append(job_finished_receipt.to_receipt()) - except Exception as e: - logger.error( - f"Skipping job finished receipt for job {job_finished_receipt.job_uuid}: {e}" - ) + for model in [JobStartedReceipt, JobAcceptedReceipt, JobFinishedReceipt]: + db_objects = model.objects.order_by("timestamp").filter( + timestamp__gt=now() - RECEIPTS_MAX_SERVED_PERIOD + ) + for db_object in db_objects: + try: + receipts.append(db_object.to_receipt()) + except Exception as e: + logger.error(f"Skipping job started receipt for job {db_object.job_uuid}: {e}") logger.info(f"Stored receipts: {len(receipts)}") @@ -99,12 +88,8 @@ def prepare_receipts(): @app.task def clear_old_receipts(): - JobFinishedReceipt.objects.filter( - time_started__lt=now() - RECEIPTS_MAX_RETENTION_PERIOD - ).delete() - JobStartedReceipt.objects.filter( - time_accepted__lt=now() - RECEIPTS_MAX_RETENTION_PERIOD - ).delete() + for model in [JobStartedReceipt, JobAcceptedReceipt, JobFinishedReceipt]: + model.objects.filter(timestamp__lt=now() - RECEIPTS_MAX_RETENTION_PERIOD).delete() @app.task @@ -124,43 +109,70 @@ def get_receipts_from_old_miner(): tolerance = datetime.timedelta(hours=1) latest_job_started_receipt = ( - JobStartedReceipt.objects.filter(miner_hotkey=hotkey).order_by("-time_accepted").first() + JobStartedReceipt.objects.filter(miner_hotkey=hotkey).order_by("-timestamp").first() ) job_started_receipt_cutoff_time = ( - latest_job_started_receipt.time_accepted - tolerance if latest_job_started_receipt else None + latest_job_started_receipt.timestamp - tolerance if latest_job_started_receipt else None ) job_started_receipt_to_create = [ JobStartedReceipt( job_uuid=receipt.payload.job_uuid, miner_hotkey=receipt.payload.miner_hotkey, validator_hotkey=receipt.payload.validator_hotkey, + timestamp=receipt.payload.timestamp, executor_class=receipt.payload.executor_class, - time_accepted=receipt.payload.time_accepted, max_timeout=receipt.payload.max_timeout, + ttl=receipt.payload.ttl, ) for receipt in receipts if isinstance(receipt.payload, JobStartedReceiptPayload) and ( job_started_receipt_cutoff_time is None - or receipt.payload.time_accepted > job_started_receipt_cutoff_time + or receipt.payload.timestamp > job_started_receipt_cutoff_time ) ] if job_started_receipt_to_create: JobStartedReceipt.objects.bulk_create(job_started_receipt_to_create, ignore_conflicts=True) + latest_job_accepted_receipt = ( + JobAcceptedReceipt.objects.filter(miner_hotkey=hotkey).order_by("-timestamp").first() + ) + job_accepted_receipt_cutoff_time = ( + latest_job_accepted_receipt.timestamp - tolerance if latest_job_accepted_receipt else None + ) + job_accepted_receipt_to_create = [ + JobAcceptedReceipt( + job_uuid=receipt.payload.job_uuid, + miner_hotkey=receipt.payload.miner_hotkey, + validator_hotkey=receipt.payload.validator_hotkey, + timestamp=receipt.payload.timestamp, + time_accepted=receipt.payload.time_accepted, + ttl=receipt.payload.ttl, + ) + for receipt in receipts + if isinstance(receipt.payload, JobAcceptedReceiptPayload) + and ( + job_accepted_receipt_cutoff_time is None + or receipt.payload.timestamp > job_accepted_receipt_cutoff_time + ) + ] + if job_accepted_receipt_to_create: + JobAcceptedReceipt.objects.bulk_create( + job_accepted_receipt_to_create, ignore_conflicts=True + ) + latest_job_finished_receipt = ( - JobFinishedReceipt.objects.filter(miner_hotkey=hotkey).order_by("-time_started").first() + JobFinishedReceipt.objects.filter(miner_hotkey=hotkey).order_by("-timestamp").first() ) job_finished_receipt_cutoff_time = ( - latest_job_finished_receipt.time_started - tolerance - if latest_job_finished_receipt - else None + latest_job_finished_receipt.timestamp - tolerance if latest_job_finished_receipt else None ) job_finished_receipt_to_create = [ JobFinishedReceipt( job_uuid=receipt.payload.job_uuid, miner_hotkey=receipt.payload.miner_hotkey, validator_hotkey=receipt.payload.validator_hotkey, + timestamp=receipt.payload.timestamp, time_started=receipt.payload.time_started, time_took_us=receipt.payload.time_took_us, score_str=receipt.payload.score_str, @@ -169,7 +181,7 @@ def get_receipts_from_old_miner(): if isinstance(receipt.payload, JobFinishedReceiptPayload) and ( job_finished_receipt_cutoff_time is None - or receipt.payload.time_started > job_finished_receipt_cutoff_time + or receipt.payload.timestamp > job_finished_receipt_cutoff_time ) ] if job_finished_receipt_to_create: From 9ec2636ba852219f9d506de6802ce4ba1b07163c Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sat, 19 Oct 2024 01:00:28 +0600 Subject: [PATCH 17/40] Fix receipts admin classes --- compute_horde/compute_horde/receipts/admin.py | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/compute_horde/compute_horde/receipts/admin.py b/compute_horde/compute_horde/receipts/admin.py index 712a15f79..411ac4641 100644 --- a/compute_horde/compute_horde/receipts/admin.py +++ b/compute_horde/compute_horde/receipts/admin.py @@ -1,7 +1,7 @@ from django.contrib import admin # noqa from compute_horde.base.admin import ReadOnlyAdminMixin -from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt, JobAcceptedReceipt class JobStartedReceiptsReadOnlyAdmin(admin.ModelAdmin, ReadOnlyAdminMixin): @@ -9,11 +9,24 @@ class JobStartedReceiptsReadOnlyAdmin(admin.ModelAdmin, ReadOnlyAdminMixin): "job_uuid", "miner_hotkey", "validator_hotkey", + "timestamp", "executor_class", - "time_accepted", "max_timeout", + "ttl", + ] + ordering = ["-timestamp"] + + +class JobAcceptedReceiptsReadOnlyAdmin(admin.ModelAdmin, ReadOnlyAdminMixin): + list_display = [ + "job_uuid", + "miner_hotkey", + "validator_hotkey", + "timestamp", + "time_accepted", + "ttl", ] - ordering = ["-time_accepted"] + ordering = ["-timestamp"] class JobFinishedReceiptsReadOnlyAdmin(admin.ModelAdmin, ReadOnlyAdminMixin): @@ -21,12 +34,14 @@ class JobFinishedReceiptsReadOnlyAdmin(admin.ModelAdmin, ReadOnlyAdminMixin): "job_uuid", "miner_hotkey", "validator_hotkey", + "timestamp", "score", "time_started", "time_took", ] - ordering = ["-time_started"] + ordering = ["-timestamp"] admin.site.register(JobStartedReceipt, admin_class=JobStartedReceiptsReadOnlyAdmin) +admin.site.register(JobAcceptedReceipt, admin_class=JobAcceptedReceiptsReadOnlyAdmin) admin.site.register(JobFinishedReceipt, admin_class=JobFinishedReceiptsReadOnlyAdmin) From fe318a04e01fb399b8623e64726640cd479080c2 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sat, 19 Oct 2024 02:19:12 +0600 Subject: [PATCH 18/40] Fix miner tests --- .../compute_horde/receipts/schemas.py | 2 +- .../test_mocked_executor_manager.py | 77 ++++++++++++++----- .../miner/tests/test_migration.py | 28 +++++-- 3 files changed, 81 insertions(+), 26 deletions(-) diff --git a/compute_horde/compute_horde/receipts/schemas.py b/compute_horde/compute_horde/receipts/schemas.py index 6d56930d8..3af0b6c42 100644 --- a/compute_horde/compute_horde/receipts/schemas.py +++ b/compute_horde/compute_horde/receipts/schemas.py @@ -9,7 +9,7 @@ from compute_horde.executor_class import ExecutorClass -class ReceiptType(enum.Enum): +class ReceiptType(enum.StrEnum): JobStartedReceipt = "JobStartedReceipt" JobAcceptedReceipt = "JobAcceptedReceipt" JobFinishedReceipt = "JobFinishedReceipt" diff --git a/miner/app/src/compute_horde_miner/miner/tests/integration/test_mocked_executor_manager.py b/miner/app/src/compute_horde_miner/miner/tests/integration/test_mocked_executor_manager.py index cd099ea5f..460e39ad2 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/integration/test_mocked_executor_manager.py +++ b/miner/app/src/compute_horde_miner/miner/tests/integration/test_mocked_executor_manager.py @@ -1,10 +1,12 @@ import contextlib +import json import time import uuid -from unittest.mock import MagicMock +from unittest.mock import Mock import pytest import pytest_asyncio +from bittensor import Keypair from channels.testing import WebsocketCommunicator from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS from pytest_mock import MockerFixture @@ -20,9 +22,11 @@ @pytest.fixture def mock_keypair(mocker: MockerFixture): - return mocker.patch( - "compute_horde_miner.miner.miner_consumer.validator_interface.bittensor.Keypair" + mock = Mock(wraps=Keypair) + mocker.patch( + "compute_horde_miner.miner.miner_consumer.validator_interface.bittensor.Keypair", mock ) + return mock # Somehow the regular dependency mechanism doesn't work with multiple test cases @@ -44,8 +48,15 @@ def job_uuid(): @pytest.fixture -def validator_hotkey(): - return "some_public_key" +def validator_keypair(): + return Keypair.create_from_mnemonic( + "slot excuse valid grief praise rifle spoil auction weasel glove pen share" + ) + + +@pytest.fixture +def validator_hotkey(validator_keypair): + return validator_keypair.ss58_address @pytest_asyncio.fixture @@ -64,17 +75,20 @@ async def make_communicator(validator_key: str): await communicator.disconnect() -async def run_regular_flow_test(validator_key: str, job_uuid: str): - async with make_communicator(validator_key) as communicator: +async def run_regular_flow_test(validator_keypair: Keypair, miner_hotkey: str, job_uuid: str): + async with make_communicator(validator_keypair.ss58_address) as communicator: + auth_payload = { + "validator_hotkey": validator_keypair.ss58_address, + "miner_hotkey": "some key", + "timestamp": int(time.time()), + } + auth_payload_blob = json.dumps(auth_payload, sort_keys=True) + auth_signature = f"0x{validator_keypair.sign(auth_payload_blob).hex()}" await communicator.send_json_to( { "message_type": "V0AuthenticateRequest", - "payload": { - "validator_hotkey": validator_key, - "miner_hotkey": "some key", - "timestamp": int(time.time()), - }, - "signature": "gibberish", + "payload": auth_payload, + "signature": auth_signature, } ) response = await communicator.receive_json_from(timeout=WEBSOCKET_TIMEOUT) @@ -84,6 +98,18 @@ async def run_regular_flow_test(validator_key: str, job_uuid: str): "executor_classes": [{"count": 1, "executor_class": DEFAULT_EXECUTOR_CLASS}] }, } + receipt = { + "receipt_type": "JobStartedReceipt", + "job_uuid": job_uuid, + "miner_hotkey": miner_hotkey, + "validator_hotkey": validator_keypair.ss58_address, + "timestamp": "2020-01-01T00:00Z", + "executor_class": DEFAULT_EXECUTOR_CLASS, + "max_timeout": 60, + "ttl": 5, + } + receipt_blob = json.dumps(receipt, sort_keys=True) + receipt_signature = f"0x{validator_keypair.sign(receipt_blob).hex()}" await communicator.send_json_to( { "message_type": "V0InitialJobRequest", @@ -92,6 +118,8 @@ async def run_regular_flow_test(validator_key: str, job_uuid: str): "base_docker_image_name": "it's teeeeests", "timeout_seconds": 60, "volume_type": "inline", + "job_started_receipt_payload": receipt, + "job_started_receipt_signature": receipt_signature, } ) response = await communicator.receive_json_from(timeout=WEBSOCKET_TIMEOUT) @@ -125,21 +153,32 @@ async def run_regular_flow_test(validator_key: str, job_uuid: str): } -async def test_main_loop(validator: Validator, job_uuid: str): - await run_regular_flow_test(validator.public_key, job_uuid) +async def test_main_loop(validator: Validator, validator_keypair: Keypair, job_uuid: str, settings): + await run_regular_flow_test( + validator_keypair, + settings.BITTENSOR_WALLET().hotkey.ss58_address, + job_uuid, + ) -async def test_local_miner(validator: Validator, job_uuid: str, mock_keypair: MagicMock, settings): +async def test_local_miner( + validator: Validator, + validator_keypair: Keypair, + job_uuid: str, + mock_keypair: Mock, + settings, +): settings.IS_LOCAL_MINER = True settings.DEBUG_TURN_AUTHENTICATION_OFF = False - await run_regular_flow_test(validator.public_key, job_uuid) + await run_regular_flow_test( + validator_keypair, settings.BITTENSOR_WALLET().hotkey.ss58_address, job_uuid + ) mock_keypair.assert_called_once_with(ss58_address=validator.public_key) - mock_keypair.return_value.verify.assert_called_once() -async def test_local_miner_unknown_validator(mock_keypair: MagicMock, settings): +async def test_local_miner_unknown_validator(mock_keypair: Mock, settings): settings.IS_LOCAL_MINER = True settings.DEBUG_TURN_AUTHENTICATION_OFF = False diff --git a/miner/app/src/compute_horde_miner/miner/tests/test_migration.py b/miner/app/src/compute_horde_miner/miner/tests/test_migration.py index 556195e33..224355579 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/test_migration.py +++ b/miner/app/src/compute_horde_miner/miner/tests/test_migration.py @@ -1,4 +1,5 @@ import uuid +from datetime import UTC, datetime import pytest from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS @@ -6,8 +7,8 @@ JobFinishedReceiptPayload, JobStartedReceiptPayload, ) -from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt -from compute_horde.receipts.schemas import Receipt +from compute_horde.receipts.models import JobAcceptedReceipt, JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.schemas import JobAcceptedReceiptPayload, Receipt from django.utils.timezone import now from pytest_mock import MockerFixture @@ -47,9 +48,22 @@ def test_get_receipts_from_old_miner(mocker: MockerFixture): job_uuid=str(uuid.uuid4()), miner_hotkey="m1", validator_hotkey="v1", + timestamp=datetime(2020, 1, 1, tzinfo=UTC), executor_class=DEFAULT_EXECUTOR_CLASS, - time_accepted=now(), max_timeout=30, + ttl=5, + ), + validator_signature="0xv1", + miner_signature="0xm1", + ), + Receipt( + payload=JobAcceptedReceiptPayload( + job_uuid=str(uuid.uuid4()), + miner_hotkey="m1", + validator_hotkey="v1", + timestamp=datetime(2020, 1, 1, tzinfo=UTC), + time_accepted=datetime(2020, 1, 1, tzinfo=UTC), + ttl=5, ), validator_signature="0xv1", miner_signature="0xm1", @@ -58,13 +72,14 @@ def test_get_receipts_from_old_miner(mocker: MockerFixture): payload=JobFinishedReceiptPayload( job_uuid=str(uuid.uuid4()), miner_hotkey="m1", - validator_hotkey="v2", + validator_hotkey="v3", + timestamp=datetime(2020, 1, 1, tzinfo=UTC), time_started=now(), time_took_us=35_000_000, score_str="103.45", ), - validator_signature="0xv2", - miner_signature="0xm2", + validator_signature="0xv3", + miner_signature="0xm3", ), ] mocker.patch("compute_horde_miner.miner.tasks.get_miner_receipts", return_value=receipts) @@ -73,4 +88,5 @@ def test_get_receipts_from_old_miner(mocker: MockerFixture): get_receipts_from_old_miner() assert JobStartedReceipt.objects.count() == 1 + assert JobAcceptedReceipt.objects.count() == 1 assert JobFinishedReceipt.objects.count() == 1 From 3c979116225ba535338dcbebb7fa515afe3166c7 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sat, 19 Oct 2024 02:26:12 +0600 Subject: [PATCH 19/40] Add receipts migration --- ...obstartedreceipt_time_accepted_and_more.py | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 compute_horde/compute_horde/receipts/migrations/0002_remove_jobstartedreceipt_time_accepted_and_more.py diff --git a/compute_horde/compute_horde/receipts/migrations/0002_remove_jobstartedreceipt_time_accepted_and_more.py b/compute_horde/compute_horde/receipts/migrations/0002_remove_jobstartedreceipt_time_accepted_and_more.py new file mode 100644 index 000000000..232c72793 --- /dev/null +++ b/compute_horde/compute_horde/receipts/migrations/0002_remove_jobstartedreceipt_time_accepted_and_more.py @@ -0,0 +1,67 @@ +# Generated by Django 5.1.1 on 2024-10-18 19:07 + +import datetime + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("receipts", "0001_initial"), + ] + + operations = [ + migrations.RemoveField( + model_name="jobstartedreceipt", + name="time_accepted", + ), + migrations.AddField( + model_name="jobfinishedreceipt", + name="timestamp", + field=models.DateTimeField( + default=datetime.datetime(2020, 1, 1, 0, 0, tzinfo=datetime.UTC) + ), + preserve_default=False, + ), + migrations.AddField( + model_name="jobstartedreceipt", + name="timestamp", + field=models.DateTimeField( + default=datetime.datetime(2020, 1, 1, 0, 0, tzinfo=datetime.UTC) + ), + preserve_default=False, + ), + migrations.AddField( + model_name="jobstartedreceipt", + name="ttl", + field=models.IntegerField(default=0), + preserve_default=False, + ), + migrations.CreateModel( + name="JobAcceptedReceipt", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("job_uuid", models.UUIDField()), + ("validator_hotkey", models.CharField(max_length=256)), + ("miner_hotkey", models.CharField(max_length=256)), + ("validator_signature", models.CharField(max_length=256)), + ("miner_signature", models.CharField(blank=True, max_length=256, null=True)), + ("timestamp", models.DateTimeField()), + ("time_accepted", models.DateTimeField()), + ("ttl", models.IntegerField()), + ], + options={ + "abstract": False, + "constraints": [ + models.UniqueConstraint( + fields=("job_uuid",), name="receipts_unique_jobacceptedreceipt_job_uuid" + ) + ], + }, + ), + ] From bf2e8977c1cb6d4c0acfa458015967fe666373a7 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 01:42:28 +0600 Subject: [PATCH 20/40] Fix surface level errors in validator --- .../validator/synthetic_jobs/batch_run.py | 54 ++++++++----------- .../validator/tasks.py | 25 +++++---- .../validator/tests/test_miner_driver.py | 6 +-- .../validator/tests/test_receipts.py | 11 ++-- 4 files changed, 45 insertions(+), 51 deletions(-) diff --git a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py index ecda12a22..450bd5c1a 100644 --- a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py +++ b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py @@ -37,15 +37,13 @@ ) from compute_horde.mv_protocol.validator_requests import ( AuthenticationPayload, - JobFinishedReceiptPayload, - JobStartedReceiptPayload, V0AuthenticateRequest, V0InitialJobRequest, V0JobFinishedReceiptRequest, V0JobRequest, - V0JobStartedReceiptRequest, ) from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.schemas import JobFinishedReceiptPayload, JobStartedReceiptPayload from compute_horde.transport import AbstractTransport, WSTransport from compute_horde.transport.base import TransportConnectionError from django.conf import settings @@ -268,7 +266,8 @@ class Job: machine_specs: V0MachineSpecsRequest | None = None # receipts - job_started_receipt: V0JobStartedReceiptRequest | None = None + job_started_receipt_payload: JobStartedReceiptPayload | None = None + job_started_receipt_signature: str | None = None job_finished_receipt: V0JobFinishedReceiptRequest | None = None # scoring @@ -685,7 +684,8 @@ def _get_total_executor_count(ctx: BatchContext) -> int: def _generate_job_started_receipt(ctx: BatchContext, job: Job) -> None: - assert job.job_started_receipt is None + assert job.job_started_receipt_payload is None + assert job.job_started_receipt_signature is None assert job.executor_response_time is not None @@ -694,14 +694,14 @@ def _generate_job_started_receipt(ctx: BatchContext, job: Job) -> None: job_uuid=job.uuid, miner_hotkey=job.miner_hotkey, validator_hotkey=ctx.own_keypair.ss58_address, + timestamp=datetime.now(tz=UTC), executor_class=ExecutorClass(job.executor_class), - time_accepted=job.executor_response_time, max_timeout=max_timeout, + ttl=30, # FIXME: use spin up time ) - job.job_started_receipt = V0JobStartedReceiptRequest( - payload=payload, - signature=f"0x{ctx.own_keypair.sign(payload.blob_for_signing()).hex()}", - ) + signature = f"0x{ctx.own_keypair.sign(payload.blob_for_signing()).hex()}" + job.job_started_receipt_payload = payload + job.job_started_receipt_signature = signature def _generate_job_finished_receipt(ctx: BatchContext, job: Job) -> None: @@ -720,6 +720,7 @@ def _generate_job_finished_receipt(ctx: BatchContext, job: Job) -> None: job_uuid=job.uuid, miner_hotkey=job.miner_hotkey, validator_hotkey=ctx.own_keypair.ss58_address, + timestamp=datetime.now(tz=UTC), time_started=job.job_before_sent_time, time_took_us=int(time_took_sec * 1_000_000), score_str=f"{job.score:.6g}", @@ -877,6 +878,8 @@ async def _send_initial_job_request( job = ctx.jobs[job_uuid] job.accept_barrier_time = barrier_time client = ctx.clients[job.miner_hotkey] + assert job.job_started_receipt_payload is not None + assert job.job_started_receipt_signature is not None spin_up_time = EXECUTOR_CLASS[job.executor_class].spin_up_time assert spin_up_time is not None @@ -890,6 +893,8 @@ async def _send_initial_job_request( base_docker_image_name=job.job_generator.base_docker_image_name(), timeout_seconds=job.job_generator.timeout_seconds(), volume=job.volume if job.job_generator.volume_in_initial_req() else None, + job_started_receipt_payload=job.job_started_receipt_payload, + job_started_receipt_signature=job.job_started_receipt_signature, ) request_json = request.model_dump_json() @@ -907,23 +912,6 @@ async def _send_initial_job_request( if isinstance(job.accept_response, V0AcceptJobRequest): await job.executor_response_event.wait() - # send the receipt from outside the timeout - if isinstance(job.executor_response, V0ExecutorReadyRequest): - _generate_job_started_receipt(ctx, job) - assert job.job_started_receipt is not None - try: - receipt_json = job.job_started_receipt.model_dump_json() - async with asyncio.timeout(_SEND_RECEIPT_TIMEOUT): - await client.send_check(receipt_json) - except (Exception, asyncio.CancelledError) as exc: - logger.warning("%s failed to send job started receipt: %r", job.name, exc) - job.system_event( - type=SystemEvent.EventType.RECEIPT_FAILURE, - subtype=SystemEvent.EventSubType.RECEIPT_SEND_ERROR, - description=repr(exc), - func="_send_initial_job_request", - ) - async def _send_job_request( ctx: BatchContext, start_barrier: asyncio.Barrier, job_uuid: str @@ -1505,16 +1493,19 @@ def _db_persist(ctx: BatchContext) -> None: job_started_receipts: list[JobStartedReceipt] = [] for job in ctx.jobs.values(): - if job.job_started_receipt is not None: - started_payload = job.job_started_receipt.payload + if ( + job.job_started_receipt_payload is not None + and job.job_started_receipt_signature is not None + ): + started_payload = job.job_started_receipt_payload job_started_receipts.append( JobStartedReceipt( job_uuid=started_payload.job_uuid, miner_hotkey=started_payload.miner_hotkey, validator_hotkey=started_payload.validator_hotkey, - validator_signature=job.job_started_receipt.signature, + validator_signature=job.job_started_receipt_signature, + timestamp=started_payload.timestamp, executor_class=started_payload.executor_class, - time_accepted=started_payload.time_accepted, max_timeout=started_payload.max_timeout, ) ) @@ -1530,6 +1521,7 @@ def _db_persist(ctx: BatchContext) -> None: miner_hotkey=finished_payload.miner_hotkey, validator_hotkey=finished_payload.validator_hotkey, validator_signature=job.job_finished_receipt.signature, + timestamp=finished_payload.timestamp, time_started=finished_payload.time_started, time_took_us=finished_payload.time_took_us, score_str=finished_payload.score_str, diff --git a/validator/app/src/compute_horde_validator/validator/tasks.py b/validator/app/src/compute_horde_validator/validator/tasks.py index f19bdcd14..83529965d 100644 --- a/validator/app/src/compute_horde_validator/validator/tasks.py +++ b/validator/app/src/compute_horde_validator/validator/tasks.py @@ -19,11 +19,11 @@ from celery.result import allow_join_result from celery.utils.log import get_task_logger from compute_horde.dynamic_config import sync_dynamic_config -from compute_horde.mv_protocol.validator_requests import ( +from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.schemas import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, ) -from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt from compute_horde.receipts.transfer import get_miner_receipts from compute_horde.utils import ValidatorListError, get_validators from constance import config @@ -1023,43 +1023,42 @@ def fetch_receipts_from_miner(hotkey: str, ip: str, port: int): tolerance = timedelta(hours=1) latest_job_started_receipt = ( - JobStartedReceipt.objects.filter(miner_hotkey=hotkey).order_by("-time_accepted").first() + JobStartedReceipt.objects.filter(miner_hotkey=hotkey).order_by("-timestamp").first() ) job_started_receipt_cutoff_time = ( - latest_job_started_receipt.time_accepted - tolerance if latest_job_started_receipt else None + latest_job_started_receipt.timestamp - tolerance if latest_job_started_receipt else None ) job_started_receipt_to_create = [ JobStartedReceipt( job_uuid=receipt.payload.job_uuid, miner_hotkey=receipt.payload.miner_hotkey, validator_hotkey=receipt.payload.validator_hotkey, + timestamp=receipt.payload.timestamp, executor_class=receipt.payload.executor_class, - time_accepted=receipt.payload.time_accepted, max_timeout=receipt.payload.max_timeout, ) for receipt in receipts if isinstance(receipt.payload, JobStartedReceiptPayload) and ( job_started_receipt_cutoff_time is None - or receipt.payload.time_accepted > job_started_receipt_cutoff_time + or receipt.payload.timestamp > job_started_receipt_cutoff_time ) ] logger.debug(f"Creating {len(job_started_receipt_to_create)} JobStartedReceipt. {hotkey=}") JobStartedReceipt.objects.bulk_create(job_started_receipt_to_create, ignore_conflicts=True) latest_job_finished_receipt = ( - JobFinishedReceipt.objects.filter(miner_hotkey=hotkey).order_by("-time_started").first() + JobFinishedReceipt.objects.filter(miner_hotkey=hotkey).order_by("-timestamp").first() ) job_finished_receipt_cutoff_time = ( - latest_job_finished_receipt.time_started - tolerance - if latest_job_finished_receipt - else None + latest_job_finished_receipt.timestamp - tolerance if latest_job_finished_receipt else None ) job_finished_receipt_to_create = [ JobFinishedReceipt( job_uuid=receipt.payload.job_uuid, miner_hotkey=receipt.payload.miner_hotkey, validator_hotkey=receipt.payload.validator_hotkey, + timestamp=receipt.payload.timestamp, time_started=receipt.payload.time_started, time_took_us=receipt.payload.time_took_us, score_str=receipt.payload.score_str, @@ -1068,7 +1067,7 @@ def fetch_receipts_from_miner(hotkey: str, ip: str, port: int): if isinstance(receipt.payload, JobFinishedReceiptPayload) and ( job_finished_receipt_cutoff_time is None - or receipt.payload.time_started > job_finished_receipt_cutoff_time + or receipt.payload.timestamp > job_finished_receipt_cutoff_time ) ] logger.debug(f"Creating {len(job_finished_receipt_to_create)} JobFinishedReceipt. {hotkey=}") @@ -1079,8 +1078,8 @@ def fetch_receipts_from_miner(hotkey: str, ip: str, port: int): def fetch_receipts(): """Fetch job receipts from the miners.""" # Delete old receipts before fetching new ones - JobStartedReceipt.objects.filter(time_accepted__lt=now() - timedelta(days=7)).delete() - JobFinishedReceipt.objects.filter(time_started__lt=now() - timedelta(days=7)).delete() + JobStartedReceipt.objects.filter(timestamp__lt=now() - timedelta(days=7)).delete() + JobFinishedReceipt.objects.filter(timestamp__lt=now() - timedelta(days=7)).delete() metagraph = bittensor.metagraph( netuid=settings.BITTENSOR_NETUID, network=settings.BITTENSOR_NETWORK diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_miner_driver.py b/validator/app/src/compute_horde_validator/validator/tests/test_miner_driver.py index 6120d4d95..fa17550a3 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_miner_driver.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_miner_driver.py @@ -12,7 +12,6 @@ ) from compute_horde.mv_protocol.validator_requests import ( V0JobFinishedReceiptRequest, - V0JobStartedReceiptRequest, ) from compute_horde_validator.validator.models import Miner @@ -163,7 +162,8 @@ async def track_job_status_updates(x): def condition(_): return True - if expected_job_started_receipt: - assert miner_client._query_sent_models(condition, V0JobStartedReceiptRequest) + # FIXME + # if expected_job_started_receipt: + # assert miner_client._query_sent_models(condition, V0JobStartedReceiptRequest) if expected_job_finished_receipt: assert miner_client._query_sent_models(condition, V0JobFinishedReceiptRequest) diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py b/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py index aa32769f0..38ad8dcc3 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py @@ -1,14 +1,15 @@ import uuid +from datetime import datetime from typing import NamedTuple import pytest from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS -from compute_horde.mv_protocol.validator_requests import ( +from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.schemas import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, + Receipt, ) -from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt -from compute_horde.receipts.schemas import Receipt from django.utils.timezone import now from compute_horde_validator.validator.models import ( @@ -46,9 +47,10 @@ def mocked_get_miner_receipts(hotkey: str, ip: str, port: int) -> list[Receipt]: job_uuid=str(uuid.uuid4()), miner_hotkey="5G9qWBzLPVVu2fCPPvg3QgPPK5JaJmJKaJha95TPHH9NZWuL", validator_hotkey="v1", + timestamp=datetime(2020, 1, 1, 0, 0), executor_class=DEFAULT_EXECUTOR_CLASS, - time_accepted=now(), max_timeout=30, + ttl=5, ), validator_signature="0xv1", miner_signature="0xm1", @@ -61,6 +63,7 @@ def mocked_get_miner_receipts(hotkey: str, ip: str, port: int) -> list[Receipt]: job_uuid=str(uuid.uuid4()), miner_hotkey="5CPhGRp4cdEG4KSui7VQixHhvN5eBUSnMYeUF5thdxm4sKtz", validator_hotkey="v1", + timestamp=datetime(2020, 1, 1, 1, 0), time_started=now(), time_took_us=30_000_000, score_str="123.45", From bb4dd47bd4e048fa327093bf5bc3a2003a6eb255 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 02:18:34 +0600 Subject: [PATCH 21/40] Add job started receipts in validator --- .../validator/synthetic_jobs/batch_run.py | 60 ++++++++++++++++++- .../validator/tasks.py | 28 ++++++++- 2 files changed, 85 insertions(+), 3 deletions(-) diff --git a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py index 450bd5c1a..fb58946b0 100644 --- a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py +++ b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py @@ -39,11 +39,16 @@ AuthenticationPayload, V0AuthenticateRequest, V0InitialJobRequest, + V0JobAcceptedReceiptRequest, V0JobFinishedReceiptRequest, V0JobRequest, ) -from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt -from compute_horde.receipts.schemas import JobFinishedReceiptPayload, JobStartedReceiptPayload +from compute_horde.receipts.models import JobAcceptedReceipt, JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.schemas import ( + JobAcceptedReceiptPayload, + JobFinishedReceiptPayload, + JobStartedReceiptPayload, +) from compute_horde.transport import AbstractTransport, WSTransport from compute_horde.transport.base import TransportConnectionError from django.conf import settings @@ -268,6 +273,7 @@ class Job: # receipts job_started_receipt_payload: JobStartedReceiptPayload | None = None job_started_receipt_signature: str | None = None + job_accepted_receipt: V0JobAcceptedReceiptRequest | None = None job_finished_receipt: V0JobFinishedReceiptRequest | None = None # scoring @@ -704,6 +710,24 @@ def _generate_job_started_receipt(ctx: BatchContext, job: Job) -> None: job.job_started_receipt_signature = signature +def _generate_job_accepted_receipt(ctx: BatchContext, job: Job) -> None: + assert job.job_accepted_receipt is None + assert job.accept_response_time is not None + + payload = JobAcceptedReceiptPayload( + job_uuid=job.uuid, + miner_hotkey=job.miner_hotkey, + validator_hotkey=ctx.own_keypair.ss58_address, + timestamp=datetime.now(tz=UTC), + time_accepted=job.accept_response_time, + ttl=300, # FIXME: max time allowed to run the job + ) + job.job_accepted_receipt = V0JobAcceptedReceiptRequest( + payload=payload, + signature=f"0x{ctx.own_keypair.sign(payload.blob_for_signing()).hex()}", + ) + + def _generate_job_finished_receipt(ctx: BatchContext, job: Job) -> None: assert job.job_finished_receipt is None assert job.job_before_sent_time is not None @@ -910,6 +934,21 @@ async def _send_initial_job_request( await job.accept_response_event.wait() if isinstance(job.accept_response, V0AcceptJobRequest): + _generate_job_accepted_receipt(ctx, job) + assert job.job_accepted_receipt is not None + try: + receipt_json = job.job_accepted_receipt.model_dump_json() + async with asyncio.timeout(_SEND_RECEIPT_TIMEOUT): + await client.send_check(receipt_json) + except (Exception, asyncio.CancelledError) as exc: + logger.warning("%s failed to send job accepted receipt: %r", job.name, exc) + job.system_event( + type=SystemEvent.EventType.RECEIPT_FAILURE, + subtype=SystemEvent.EventSubType.RECEIPT_SEND_ERROR, + description=repr(exc), + func="_send_initial_job_request", + ) + await job.executor_response_event.wait() @@ -1511,6 +1550,23 @@ def _db_persist(ctx: BatchContext) -> None: ) JobStartedReceipt.objects.bulk_create(job_started_receipts) + job_accepted_receipts: list[JobAcceptedReceipt] = [] + for job in ctx.jobs.values(): + if job.job_accepted_receipt is not None: + accepted_payload = job.job_accepted_receipt.payload + job_accepted_receipts.append( + JobAcceptedReceipt( + job_uuid=accepted_payload.job_uuid, + miner_hotkey=accepted_payload.miner_hotkey, + validator_hotkey=accepted_payload.validator_hotkey, + validator_signature=job.job_accepted_receipt.signature, + timestamp=accepted_payload.timestamp, + time_accepted=accepted_payload.time_accepted, + ttl=accepted_payload.ttl, + ) + ) + JobAcceptedReceipt.objects.bulk_create(job_accepted_receipts) + job_finished_receipts: list[JobFinishedReceipt] = [] for job in ctx.jobs.values(): if job.job_finished_receipt is not None: diff --git a/validator/app/src/compute_horde_validator/validator/tasks.py b/validator/app/src/compute_horde_validator/validator/tasks.py index 83529965d..e5090d703 100644 --- a/validator/app/src/compute_horde_validator/validator/tasks.py +++ b/validator/app/src/compute_horde_validator/validator/tasks.py @@ -19,8 +19,9 @@ from celery.result import allow_join_result from celery.utils.log import get_task_logger from compute_horde.dynamic_config import sync_dynamic_config -from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.models import JobAcceptedReceipt, JobFinishedReceipt, JobStartedReceipt from compute_horde.receipts.schemas import ( + JobAcceptedReceiptPayload, JobFinishedReceiptPayload, JobStartedReceiptPayload, ) @@ -1047,6 +1048,30 @@ def fetch_receipts_from_miner(hotkey: str, ip: str, port: int): logger.debug(f"Creating {len(job_started_receipt_to_create)} JobStartedReceipt. {hotkey=}") JobStartedReceipt.objects.bulk_create(job_started_receipt_to_create, ignore_conflicts=True) + latest_job_accepted_receipt = ( + JobAcceptedReceipt.objects.filter(miner_hotkey=hotkey).order_by("-timestamp").first() + ) + job_accepted_receipt_cutoff_time = ( + latest_job_accepted_receipt.timestamp - tolerance if latest_job_accepted_receipt else None + ) + job_accepted_receipt_to_create = [ + JobAcceptedReceipt( + job_uuid=receipt.payload.job_uuid, + miner_hotkey=receipt.payload.miner_hotkey, + validator_hotkey=receipt.payload.validator_hotkey, + timestamp=receipt.payload.timestamp, + ttl=receipt.payload.ttl, + ) + for receipt in receipts + if isinstance(receipt.payload, JobAcceptedReceiptPayload) + and ( + job_accepted_receipt_cutoff_time is None + or receipt.payload.timestamp > job_accepted_receipt_cutoff_time + ) + ] + logger.debug(f"Creating {len(job_accepted_receipt_to_create)} JobAcceptedReceipt. {hotkey=}") + JobAcceptedReceipt.objects.bulk_create(job_accepted_receipt_to_create, ignore_conflicts=True) + latest_job_finished_receipt = ( JobFinishedReceipt.objects.filter(miner_hotkey=hotkey).order_by("-timestamp").first() ) @@ -1079,6 +1104,7 @@ def fetch_receipts(): """Fetch job receipts from the miners.""" # Delete old receipts before fetching new ones JobStartedReceipt.objects.filter(timestamp__lt=now() - timedelta(days=7)).delete() + JobAcceptedReceipt.objects.filter(timestamp__lt=now() - timedelta(days=7)).delete() JobFinishedReceipt.objects.filter(timestamp__lt=now() - timedelta(days=7)).delete() metagraph = bittensor.metagraph( From 8d2b3139ae62490610de49e5c1c175a83e7627ab Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 02:55:15 +0600 Subject: [PATCH 22/40] Fix stupid mistakes! -_- --- .../validator/synthetic_jobs/batch_run.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py index fb58946b0..362010963 100644 --- a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py +++ b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py @@ -693,8 +693,6 @@ def _generate_job_started_receipt(ctx: BatchContext, job: Job) -> None: assert job.job_started_receipt_payload is None assert job.job_started_receipt_signature is None - assert job.executor_response_time is not None - max_timeout = job.job_generator.timeout_seconds() payload = JobStartedReceiptPayload( job_uuid=job.uuid, @@ -902,6 +900,8 @@ async def _send_initial_job_request( job = ctx.jobs[job_uuid] job.accept_barrier_time = barrier_time client = ctx.clients[job.miner_hotkey] + + _generate_job_started_receipt(ctx, job) assert job.job_started_receipt_payload is not None assert job.job_started_receipt_signature is not None @@ -1546,6 +1546,7 @@ def _db_persist(ctx: BatchContext) -> None: timestamp=started_payload.timestamp, executor_class=started_payload.executor_class, max_timeout=started_payload.max_timeout, + ttl=started_payload.ttl, ) ) JobStartedReceipt.objects.bulk_create(job_started_receipts) From dd0611986cfe5f29786343fb7779ba5f06186c82 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 03:19:49 +0600 Subject: [PATCH 23/40] Fix lib tests --- .../compute_horde/receipts/models.py | 6 +-- compute_horde/tests/conftest.py | 29 ++++++++++++--- compute_horde/tests/test_receipts.py | 37 +++++++++++-------- compute_horde/tests/test_run_organic_job.py | 4 +- .../validator/tests/test_receipts.py | 6 +-- 5 files changed, 53 insertions(+), 29 deletions(-) diff --git a/compute_horde/compute_horde/receipts/models.py b/compute_horde/compute_horde/receipts/models.py index 9298f408b..1353a9885 100644 --- a/compute_horde/compute_horde/receipts/models.py +++ b/compute_horde/compute_horde/receipts/models.py @@ -23,9 +23,6 @@ class AbstractReceipt(models.Model): miner_signature = models.CharField(max_length=256, null=True, blank=True) timestamp = models.DateTimeField() - # https://github.com/typeddjango/django-stubs/issues/1684#issuecomment-1706446344 - objects: models.Manager["JobStartedReceipt"] - class Meta: abstract = True constraints = [ @@ -67,6 +64,9 @@ class JobAcceptedReceipt(AbstractReceipt): time_accepted = models.DateTimeField() ttl = models.IntegerField() + # https://github.com/typeddjango/django-stubs/issues/1684#issuecomment-1706446344 + objects: models.Manager["JobAcceptedReceipt"] + def to_receipt(self) -> Receipt: if self.miner_signature is None: raise ReceiptNotSigned("Miner signature is required") diff --git a/compute_horde/tests/conftest.py b/compute_horde/tests/conftest.py index cc4fd69b7..02946164a 100644 --- a/compute_horde/tests/conftest.py +++ b/compute_horde/tests/conftest.py @@ -6,6 +6,7 @@ from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS from compute_horde.receipts.schemas import ( + JobAcceptedReceiptPayload, JobFinishedReceiptPayload, JobStartedReceiptPayload, Receipt, @@ -39,9 +40,10 @@ def receipts(validator_keypair, miner_keypair): job_uuid="3342460e-4a99-438b-8757-795f4cb348dd", miner_hotkey=miner_keypair.ss58_address, validator_hotkey=validator_keypair.ss58_address, + timestamp=datetime.datetime(2020, 1, 1, 0, 0, 0, tzinfo=datetime.UTC), executor_class=DEFAULT_EXECUTOR_CLASS, - time_accepted=datetime.datetime(2024, 1, 2, 1, 55, 0, tzinfo=datetime.UTC), max_timeout=30, + ttl=30, ) receipt1 = Receipt( payload=payload1, @@ -49,13 +51,13 @@ def receipts(validator_keypair, miner_keypair): miner_signature=f"0x{miner_keypair.sign(payload1.blob_for_signing()).hex()}", ) - payload2 = JobFinishedReceiptPayload( + payload2 = JobAcceptedReceiptPayload( job_uuid="3342460e-4a99-438b-8757-795f4cb348dd", miner_hotkey=miner_keypair.ss58_address, validator_hotkey=validator_keypair.ss58_address, - time_started=datetime.datetime(2024, 1, 2, 1, 57, 0, tzinfo=datetime.UTC), - time_took_us=2_000_000, - score_str="2.00", + timestamp=datetime.datetime(2020, 1, 1, 0, 5, 0, tzinfo=datetime.UTC), + time_accepted=datetime.datetime(2020, 1, 1, 0, 4, 0, tzinfo=datetime.UTC), + ttl=300, ) receipt2 = Receipt( payload=payload2, @@ -63,7 +65,22 @@ def receipts(validator_keypair, miner_keypair): miner_signature=f"0x{miner_keypair.sign(payload2.blob_for_signing()).hex()}", ) - return [receipt1, receipt2] + payload3 = JobFinishedReceiptPayload( + job_uuid="3342460e-4a99-438b-8757-795f4cb348dd", + miner_hotkey=miner_keypair.ss58_address, + validator_hotkey=validator_keypair.ss58_address, + timestamp=datetime.datetime(2020, 1, 1, 0, 10, 0, tzinfo=datetime.UTC), + time_started=datetime.datetime(2020, 1, 1, 0, 9, 0, tzinfo=datetime.UTC), + time_took_us=60_000_000, + score_str="2.00", + ) + receipt3 = Receipt( + payload=payload3, + validator_signature=f"0x{validator_keypair.sign(payload3.blob_for_signing()).hex()}", + miner_signature=f"0x{miner_keypair.sign(payload3.blob_for_signing()).hex()}", + ) + + return [receipt1, receipt2, receipt3] @pytest.fixture diff --git a/compute_horde/tests/test_receipts.py b/compute_horde/tests/test_receipts.py index baa8619b5..a4877a6bd 100644 --- a/compute_horde/tests/test_receipts.py +++ b/compute_horde/tests/test_receipts.py @@ -4,17 +4,21 @@ import pytest from compute_horde.receipts.schemas import ( + JobAcceptedReceiptPayload, JobFinishedReceiptPayload, JobStartedReceiptPayload, Receipt, - ReceiptType, ) from compute_horde.receipts.transfer import ReceiptFetchError, get_miner_receipts def receipts_helper(mocked_responses, receipts: list[Receipt], miner_keypair): payload_fields = set() - for payload_cls in [JobStartedReceiptPayload, JobFinishedReceiptPayload]: + for payload_cls in [ + JobStartedReceiptPayload, + JobAcceptedReceiptPayload, + JobFinishedReceiptPayload, + ]: payload_fields |= set(payload_cls.model_fields.keys()) buf = io.StringIO() @@ -29,11 +33,7 @@ def receipts_helper(mocked_responses, receipts: list[Receipt], miner_keypair): ) csv_writer.writeheader() for receipt in receipts: - match receipt.payload: - case JobStartedReceiptPayload(): - receipt_type = ReceiptType.JobStartedReceipt - case JobFinishedReceiptPayload(): - receipt_type = ReceiptType.JobFinishedReceipt + receipt_type = receipt.payload.receipt_type row = ( dict( type=receipt_type.value, @@ -51,13 +51,20 @@ def receipts_helper(mocked_responses, receipts: list[Receipt], miner_keypair): def receipts_one_skipped_helper(mocked_responses, receipts, miner_keypair): got_receipts = receipts_helper(mocked_responses, receipts, miner_keypair) # only the valid receipt should be stored - assert len(got_receipts) == 1 - assert got_receipts[0] == receipts[0] + assert len(got_receipts) == len(receipts) - 1 + for receipt in receipts[1:]: + got_receipt = [ + x + for x in got_receipts + if x.payload.job_uuid == receipt.payload.job_uuid + and x.payload.__class__ is receipt.payload.__class__ + ][0] + assert got_receipt == receipt def test__get_miner_receipts__happy_path(mocked_responses, receipts, miner_keypair): got_receipts = receipts_helper(mocked_responses, receipts, miner_keypair) - assert len(got_receipts) == 2 + assert len(got_receipts) == len(receipts) for receipt in receipts: got_receipt = [ x @@ -74,29 +81,29 @@ def test__get_miner_receipts__invalid_receipt_skipped(mocked_responses, receipts Invalidate one receipt payload fields to make it invalid """ - receipts[1].payload.miner_hotkey = 0 - receipts[1].payload.validator_hotkey = None + receipts[0].payload.miner_hotkey = 0 + receipts[0].payload.validator_hotkey = None receipts_one_skipped_helper(mocked_responses, receipts, miner_keypair) def test__get_miner_receipts__miner_hotkey_mismatch_skipped( mocked_responses, receipts, miner_keypair, keypair ): - receipts[1].payload.miner_hotkey = keypair.ss58_address + receipts[0].payload.miner_hotkey = keypair.ss58_address receipts_one_skipped_helper(mocked_responses, receipts, miner_keypair) def test__get_miner_receipts__invalid_miner_signature_skipped( mocked_responses, receipts, miner_keypair ): - receipts[1].miner_signature = f"0x{miner_keypair.sign('bla').hex()}" + receipts[0].miner_signature = f"0x{miner_keypair.sign('bla').hex()}" receipts_one_skipped_helper(mocked_responses, receipts, miner_keypair) def test__get_miner_receipts__invalid_validator_signature_skipped( mocked_responses, receipts, miner_keypair ): - receipts[1].validator_signature = f"0x{miner_keypair.sign('bla').hex()}" + receipts[0].validator_signature = f"0x{miner_keypair.sign('bla').hex()}" receipts_one_skipped_helper(mocked_responses, receipts, miner_keypair) diff --git a/compute_horde/tests/test_run_organic_job.py b/compute_horde/tests/test_run_organic_job.py index ad43f8555..f28972590 100644 --- a/compute_horde/tests/test_run_organic_job.py +++ b/compute_horde/tests/test_run_organic_job.py @@ -14,9 +14,9 @@ BaseValidatorRequest, V0AuthenticateRequest, V0InitialJobRequest, + V0JobAcceptedReceiptRequest, V0JobFinishedReceiptRequest, V0JobRequest, - V0JobStartedReceiptRequest, ) from compute_horde.transport import StubTransport @@ -65,7 +65,7 @@ async def test_run_organic_job__success(keypair): assert sent_models_types == [ V0AuthenticateRequest, V0InitialJobRequest, - V0JobStartedReceiptRequest, + V0JobAcceptedReceiptRequest, V0JobRequest, V0JobFinishedReceiptRequest, ] diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py b/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py index 38ad8dcc3..ff9179f82 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py @@ -1,5 +1,5 @@ import uuid -from datetime import datetime +from datetime import datetime, UTC from typing import NamedTuple import pytest @@ -47,7 +47,7 @@ def mocked_get_miner_receipts(hotkey: str, ip: str, port: int) -> list[Receipt]: job_uuid=str(uuid.uuid4()), miner_hotkey="5G9qWBzLPVVu2fCPPvg3QgPPK5JaJmJKaJha95TPHH9NZWuL", validator_hotkey="v1", - timestamp=datetime(2020, 1, 1, 0, 0), + timestamp=datetime(2020, 1, 1, 0, 0, tzinfo=UTC), executor_class=DEFAULT_EXECUTOR_CLASS, max_timeout=30, ttl=5, @@ -63,7 +63,7 @@ def mocked_get_miner_receipts(hotkey: str, ip: str, port: int) -> list[Receipt]: job_uuid=str(uuid.uuid4()), miner_hotkey="5CPhGRp4cdEG4KSui7VQixHhvN5eBUSnMYeUF5thdxm4sKtz", validator_hotkey="v1", - timestamp=datetime(2020, 1, 1, 1, 0), + timestamp=datetime(2020, 1, 1, 1, 0, tzinfo=UTC), time_started=now(), time_took_us=30_000_000, score_str="123.45", From 2b8e95d34319e7625e4693c1db1b45a8fb717476 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 03:20:20 +0600 Subject: [PATCH 24/40] Fix when accepted receipt is sent in organic job --- compute_horde/compute_horde/miner_client/organic.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/compute_horde/compute_horde/miner_client/organic.py b/compute_horde/compute_horde/miner_client/organic.py index 32fd456ae..561ce76e5 100644 --- a/compute_horde/compute_horde/miner_client/organic.py +++ b/compute_horde/compute_horde/miner_client/organic.py @@ -427,6 +427,11 @@ async def run_organic_job( await client.notify_job_accepted(initial_response) + await client.send_job_accepted_receipt_message( + accepted_timestamp=time.time(), + ttl=int(job_timer.time_left()), + ) + try: executor_readiness_response = await asyncio.wait_for( client.executor_ready_or_failed_future, @@ -439,11 +444,6 @@ async def run_organic_job( await client.notify_executor_ready(executor_readiness_response) - await client.send_job_accepted_receipt_message( - accepted_timestamp=time.time(), - ttl=int(job_timer.time_left()), - ) - await client.send_model( V0JobRequest( job_uuid=job_details.job_uuid, From f343613a2a44c119930cbc47fa175b31d3a60c3a Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 03:40:49 +0600 Subject: [PATCH 25/40] Fix receipts missing fields in validator db --- .../app/src/compute_horde_validator/validator/tasks.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/validator/app/src/compute_horde_validator/validator/tasks.py b/validator/app/src/compute_horde_validator/validator/tasks.py index e5090d703..2c76795bc 100644 --- a/validator/app/src/compute_horde_validator/validator/tasks.py +++ b/validator/app/src/compute_horde_validator/validator/tasks.py @@ -1034,9 +1034,12 @@ def fetch_receipts_from_miner(hotkey: str, ip: str, port: int): job_uuid=receipt.payload.job_uuid, miner_hotkey=receipt.payload.miner_hotkey, validator_hotkey=receipt.payload.validator_hotkey, + miner_signature=receipt.miner_signature, + validator_signature=receipt.validator_signature, timestamp=receipt.payload.timestamp, executor_class=receipt.payload.executor_class, max_timeout=receipt.payload.max_timeout, + ttl=receipt.payload.ttl, ) for receipt in receipts if isinstance(receipt.payload, JobStartedReceiptPayload) @@ -1059,6 +1062,8 @@ def fetch_receipts_from_miner(hotkey: str, ip: str, port: int): job_uuid=receipt.payload.job_uuid, miner_hotkey=receipt.payload.miner_hotkey, validator_hotkey=receipt.payload.validator_hotkey, + miner_signature=receipt.miner_signature, + validator_signature=receipt.validator_signature, timestamp=receipt.payload.timestamp, ttl=receipt.payload.ttl, ) @@ -1083,6 +1088,8 @@ def fetch_receipts_from_miner(hotkey: str, ip: str, port: int): job_uuid=receipt.payload.job_uuid, miner_hotkey=receipt.payload.miner_hotkey, validator_hotkey=receipt.payload.validator_hotkey, + miner_signature=receipt.miner_signature, + validator_signature=receipt.validator_signature, timestamp=receipt.payload.timestamp, time_started=receipt.payload.time_started, time_took_us=receipt.payload.time_took_us, From 234870de0790535f6a3007426615947aa98b9f5e Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 03:46:17 +0600 Subject: [PATCH 26/40] rufFFFFFF --- .../compute_horde_validator/validator/tests/test_receipts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py b/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py index ff9179f82..4f816ad55 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py @@ -1,5 +1,5 @@ import uuid -from datetime import datetime, UTC +from datetime import UTC, datetime from typing import NamedTuple import pytest From 55eb66c20f1dddac76bb0a6bdba8e5328ccea895 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 15:12:36 +0600 Subject: [PATCH 27/40] Ignore `AbstractReceipt.objects` type hints --- miner/app/src/compute_horde_miner/miner/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/miner/app/src/compute_horde_miner/miner/tasks.py b/miner/app/src/compute_horde_miner/miner/tasks.py index cc7edf44f..3b0544499 100644 --- a/miner/app/src/compute_horde_miner/miner/tasks.py +++ b/miner/app/src/compute_horde_miner/miner/tasks.py @@ -72,7 +72,7 @@ def prepare_receipts(): receipts = [] for model in [JobStartedReceipt, JobAcceptedReceipt, JobFinishedReceipt]: - db_objects = model.objects.order_by("timestamp").filter( + db_objects = model.objects.order_by("timestamp").filter( # type: ignore[attr-defined] timestamp__gt=now() - RECEIPTS_MAX_SERVED_PERIOD ) for db_object in db_objects: @@ -89,7 +89,7 @@ def prepare_receipts(): @app.task def clear_old_receipts(): for model in [JobStartedReceipt, JobAcceptedReceipt, JobFinishedReceipt]: - model.objects.filter(timestamp__lt=now() - RECEIPTS_MAX_RETENTION_PERIOD).delete() + model.objects.filter(timestamp__lt=now() - RECEIPTS_MAX_RETENTION_PERIOD).delete() # type: ignore[attr-defined] @app.task From 153987231e2526824325776b2756fd254a8aea47 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 16:04:29 +0600 Subject: [PATCH 28/40] Fix remaining miner test --- .../test_mocked_executor_manager.py | 88 +++++++------------ 1 file changed, 30 insertions(+), 58 deletions(-) diff --git a/miner/app/src/compute_horde_miner/miner/tests/integration/test_mocked_executor_manager.py b/miner/app/src/compute_horde_miner/miner/tests/integration/test_mocked_executor_manager.py index 460e39ad2..311da15ab 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/integration/test_mocked_executor_manager.py +++ b/miner/app/src/compute_horde_miner/miner/tests/integration/test_mocked_executor_manager.py @@ -1,12 +1,10 @@ import contextlib -import json import time import uuid -from unittest.mock import Mock +from unittest.mock import MagicMock import pytest import pytest_asyncio -from bittensor import Keypair from channels.testing import WebsocketCommunicator from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS from pytest_mock import MockerFixture @@ -22,11 +20,9 @@ @pytest.fixture def mock_keypair(mocker: MockerFixture): - mock = Mock(wraps=Keypair) - mocker.patch( - "compute_horde_miner.miner.miner_consumer.validator_interface.bittensor.Keypair", mock + return mocker.patch( + "compute_horde_miner.miner.miner_consumer.validator_interface.bittensor.Keypair" ) - return mock # Somehow the regular dependency mechanism doesn't work with multiple test cases @@ -48,15 +44,8 @@ def job_uuid(): @pytest.fixture -def validator_keypair(): - return Keypair.create_from_mnemonic( - "slot excuse valid grief praise rifle spoil auction weasel glove pen share" - ) - - -@pytest.fixture -def validator_hotkey(validator_keypair): - return validator_keypair.ss58_address +def validator_hotkey(): + return "some_public_key" @pytest_asyncio.fixture @@ -75,20 +64,17 @@ async def make_communicator(validator_key: str): await communicator.disconnect() -async def run_regular_flow_test(validator_keypair: Keypair, miner_hotkey: str, job_uuid: str): - async with make_communicator(validator_keypair.ss58_address) as communicator: - auth_payload = { - "validator_hotkey": validator_keypair.ss58_address, - "miner_hotkey": "some key", - "timestamp": int(time.time()), - } - auth_payload_blob = json.dumps(auth_payload, sort_keys=True) - auth_signature = f"0x{validator_keypair.sign(auth_payload_blob).hex()}" +async def run_regular_flow_test(validator_key: str, job_uuid: str): + async with make_communicator(validator_key) as communicator: await communicator.send_json_to( { "message_type": "V0AuthenticateRequest", - "payload": auth_payload, - "signature": auth_signature, + "payload": { + "validator_hotkey": validator_key, + "miner_hotkey": "some key", + "timestamp": int(time.time()), + }, + "signature": "gibberish", } ) response = await communicator.receive_json_from(timeout=WEBSOCKET_TIMEOUT) @@ -98,18 +84,6 @@ async def run_regular_flow_test(validator_keypair: Keypair, miner_hotkey: str, j "executor_classes": [{"count": 1, "executor_class": DEFAULT_EXECUTOR_CLASS}] }, } - receipt = { - "receipt_type": "JobStartedReceipt", - "job_uuid": job_uuid, - "miner_hotkey": miner_hotkey, - "validator_hotkey": validator_keypair.ss58_address, - "timestamp": "2020-01-01T00:00Z", - "executor_class": DEFAULT_EXECUTOR_CLASS, - "max_timeout": 60, - "ttl": 5, - } - receipt_blob = json.dumps(receipt, sort_keys=True) - receipt_signature = f"0x{validator_keypair.sign(receipt_blob).hex()}" await communicator.send_json_to( { "message_type": "V0InitialJobRequest", @@ -118,8 +92,17 @@ async def run_regular_flow_test(validator_keypair: Keypair, miner_hotkey: str, j "base_docker_image_name": "it's teeeeests", "timeout_seconds": 60, "volume_type": "inline", - "job_started_receipt_payload": receipt, - "job_started_receipt_signature": receipt_signature, + "job_started_receipt_payload": { + "receipt_type": "JobStartedReceipt", + "job_uuid": job_uuid, + "miner_hotkey": "miner_hotkey", + "validator_hotkey": validator_key, + "timestamp": "2020-01-01T00:00Z", + "executor_class": DEFAULT_EXECUTOR_CLASS, + "max_timeout": 60, + "ttl": 5, + }, + "job_started_receipt_signature": "gibberish", } ) response = await communicator.receive_json_from(timeout=WEBSOCKET_TIMEOUT) @@ -153,32 +136,21 @@ async def run_regular_flow_test(validator_keypair: Keypair, miner_hotkey: str, j } -async def test_main_loop(validator: Validator, validator_keypair: Keypair, job_uuid: str, settings): - await run_regular_flow_test( - validator_keypair, - settings.BITTENSOR_WALLET().hotkey.ss58_address, - job_uuid, - ) +async def test_main_loop(validator: Validator, job_uuid: str, mock_keypair: MagicMock): + await run_regular_flow_test(validator.public_key, job_uuid) -async def test_local_miner( - validator: Validator, - validator_keypair: Keypair, - job_uuid: str, - mock_keypair: Mock, - settings, -): +async def test_local_miner(validator: Validator, job_uuid: str, mock_keypair: MagicMock, settings): settings.IS_LOCAL_MINER = True settings.DEBUG_TURN_AUTHENTICATION_OFF = False - await run_regular_flow_test( - validator_keypair, settings.BITTENSOR_WALLET().hotkey.ss58_address, job_uuid - ) + await run_regular_flow_test(validator.public_key, job_uuid) mock_keypair.assert_called_once_with(ss58_address=validator.public_key) + mock_keypair.return_value.verify.assert_called_once() -async def test_local_miner_unknown_validator(mock_keypair: Mock, settings): +async def test_local_miner_unknown_validator(mock_keypair: MagicMock, settings): settings.IS_LOCAL_MINER = True settings.DEBUG_TURN_AUTHENTICATION_OFF = False From 208013cb0407393ee424ecbb647e3c820dc9efe9 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 16:43:11 +0600 Subject: [PATCH 29/40] Add fixture to create miner test wallet --- .../miner/tests/conftest.py | 15 +++++++++++++++ .../miner/tests/settings.py | 19 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/miner/app/src/compute_horde_miner/miner/tests/conftest.py b/miner/app/src/compute_horde_miner/miner/tests/conftest.py index 7149037f2..63a8facca 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/conftest.py +++ b/miner/app/src/compute_horde_miner/miner/tests/conftest.py @@ -1,10 +1,25 @@ +import logging from collections.abc import Generator +import bittensor import pytest +logger = logging.getLogger(__name__) + @pytest.fixture def some() -> Generator[int, None, None]: # setup code yield 1 # teardown code + + +@pytest.fixture(scope="session", autouse=True) +def wallet(): + wallet = bittensor.wallet(name="test_miner") + try: + # workaround the overwrite flag + wallet.regenerate_coldkey(seed="0" * 64, use_password=False, overwrite=True) + wallet.regenerate_hotkey(seed="1" * 64, use_password=False, overwrite=True) + except Exception as e: + logger.error(f"Failed to create wallet: {e}") diff --git a/miner/app/src/compute_horde_miner/miner/tests/settings.py b/miner/app/src/compute_horde_miner/miner/tests/settings.py index c3f29f844..e52a0333e 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/settings.py +++ b/miner/app/src/compute_horde_miner/miner/tests/settings.py @@ -1,4 +1,7 @@ import os +import pathlib + +import bittensor os.environ.update( { @@ -13,3 +16,19 @@ EXECUTOR_MANAGER_CLASS_PATH = "compute_horde_miner.miner.tests.executor_manager:StubExecutorManager" DEBUG_TURN_AUTHENTICATION_OFF = True + +BITTENSOR_WALLET_DIRECTORY = pathlib.Path("~").expanduser() / ".bittensor" / "wallets" +BITTENSOR_WALLET_NAME = "test_miner" +BITTENSOR_WALLET_HOTKEY_NAME = "default" + + +def BITTENSOR_WALLET() -> bittensor.wallet: # type: ignore + if not BITTENSOR_WALLET_NAME or not BITTENSOR_WALLET_HOTKEY_NAME: + raise RuntimeError("Wallet not configured") + wallet = bittensor.wallet( + name=BITTENSOR_WALLET_NAME, + hotkey=BITTENSOR_WALLET_HOTKEY_NAME, + path=str(BITTENSOR_WALLET_DIRECTORY), + ) + wallet.hotkey_file.get_keypair() # this raises errors if the keys are inaccessible + return wallet From 16d4f16a776ef00d2f39a1a2fa61ffe27f56f117 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 17:27:55 +0600 Subject: [PATCH 30/40] Fix integration test --- .../test_miner_on_dev_executor_manager.py | 50 ++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/test_miner_on_dev_executor_manager.py b/tests/integration_tests/test_miner_on_dev_executor_manager.py index 591970e7d..ec8c1d852 100644 --- a/tests/integration_tests/test_miner_on_dev_executor_manager.py +++ b/tests/integration_tests/test_miner_on_dev_executor_manager.py @@ -1,3 +1,4 @@ +from datetime import datetime, UTC import asyncio import base64 import io @@ -11,6 +12,8 @@ import uuid import zipfile from unittest import mock +import bittensor +import logging import pytest import requests @@ -21,7 +24,29 @@ MINER_PORT = 8045 WEBSOCKET_TIMEOUT = 10 -validator_key = str(uuid.uuid4()) +logger = logging.getLogger(__name__) + + +def get_miner_wallet(): + wallet = bittensor.wallet(name="test_miner") + try: + # workaround the overwrite flag + wallet.regenerate_coldkey(seed="0" * 64, use_password=False, overwrite=True) + wallet.regenerate_hotkey(seed="1" * 64, use_password=False, overwrite=True) + except Exception as e: + logger.error(f"Failed to create wallet: {e}") + return wallet + + +def get_validator_wallet(): + wallet = bittensor.wallet(name="test_validator") + try: + # workaround the overwrite flag + wallet.regenerate_coldkey(seed="0" * 64, use_password=False, overwrite=True) + wallet.regenerate_hotkey(seed="1" * 64, use_password=False, overwrite=True) + except Exception as e: + logger.error(f"Failed to create wallet: {e}") + return wallet class Test(ActiveSubnetworkBaseTest): @@ -43,6 +68,7 @@ def miner_path_and_args(cls) -> list[str]: @classmethod def miner_preparation_tasks(cls): + validator_key = get_validator_wallet().get_hotkey().ss58_address db_shell_cmd = f"{sys.executable} miner/app/src/manage.py dbshell" for cmd in [ f'echo "DROP DATABASE IF EXISTS compute_horde_miner_integration_test" | {db_shell_cmd}', @@ -69,6 +95,7 @@ def miner_environ(cls) -> dict[str, str]: "PORT_FOR_EXECUTORS": str(MINER_PORT), "DATABASE_SUFFIX": "_integration_test", "DEBUG_TURN_AUTHENTICATION_OFF": "1", + "BITTENSOR_WALLET_NAME": "test_miner", } @classmethod @@ -77,11 +104,16 @@ def validator_path_and_args(cls) -> list[str]: @classmethod def validator_environ(cls) -> dict[str, str]: - return {} + return { + "BITTENSOR_WALLET_NAME": "test_validator", + } @pytest.mark.asyncio async def test_echo_image(self): job_uuid = str(uuid.uuid4()) + miner_key = get_miner_wallet().get_hotkey().ss58_address + validator_wallet = get_validator_wallet() + validator_key = validator_wallet.get_hotkey().ss58_address payload = "".join( random.choice(string.ascii_uppercase + string.digits) for _ in range(32) @@ -121,6 +153,18 @@ async def test_echo_image(self): ] }, } + + receipt_payload = { + "job_uuid": job_uuid, + "miner_hotkey": miner_key, + "validator_hotkey": validator_key, + "timestamp": datetime.now(tz=UTC).isoformat(), + "executor_class": DEFAULT_EXECUTOR_CLASS, + "max_timeout": 60, + "ttl": 30, + } + blob = json.dumps(receipt_payload, sort_keys=True) + signature = "0x" + validator_wallet.get_hotkey().sign(blob).hex() await ws.send( json.dumps( { @@ -130,6 +174,8 @@ async def test_echo_image(self): "base_docker_image_name": "backenddevelopersltd/compute-horde-job-echo:v0-latest", "timeout_seconds": 60, "volume_type": "inline", + "job_started_receipt_payload": receipt_payload, + "job_started_receipt_signature": signature, } ) ) From aa75e6c1fd7eb01fb20b8dd68362f1e21193a36d Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 17:51:45 +0600 Subject: [PATCH 31/40] Fix job started receipt ttl --- .../validator/synthetic_jobs/batch_run.py | 15 +++++++++------ .../validator/tests/test_miner_driver.py | 12 ++++++------ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py index 362010963..498d8f6ac 100644 --- a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py +++ b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py @@ -391,6 +391,12 @@ def emit_telemetry_event(self) -> SystemEvent | None: data=data, ) + def get_spin_up_time(self) -> int: + spin_up_time = EXECUTOR_CLASS[self.executor_class].spin_up_time + assert spin_up_time is not None + spin_up_time = max(spin_up_time, _MIN_SPIN_UP_TIME) + return spin_up_time + @dataclass class BatchContext: @@ -701,7 +707,7 @@ def _generate_job_started_receipt(ctx: BatchContext, job: Job) -> None: timestamp=datetime.now(tz=UTC), executor_class=ExecutorClass(job.executor_class), max_timeout=max_timeout, - ttl=30, # FIXME: use spin up time + ttl=job.get_spin_up_time(), ) signature = f"0x{ctx.own_keypair.sign(payload.blob_for_signing()).hex()}" job.job_started_receipt_payload = payload @@ -718,7 +724,7 @@ def _generate_job_accepted_receipt(ctx: BatchContext, job: Job) -> None: validator_hotkey=ctx.own_keypair.ss58_address, timestamp=datetime.now(tz=UTC), time_accepted=job.accept_response_time, - ttl=300, # FIXME: max time allowed to run the job + ttl=6 * 60, # FIXME: max time allowed to run the job ) job.job_accepted_receipt = V0JobAcceptedReceiptRequest( payload=payload, @@ -905,10 +911,7 @@ async def _send_initial_job_request( assert job.job_started_receipt_payload is not None assert job.job_started_receipt_signature is not None - spin_up_time = EXECUTOR_CLASS[job.executor_class].spin_up_time - assert spin_up_time is not None - spin_up_time = max(spin_up_time, _MIN_SPIN_UP_TIME) - stagger_wait_interval = max_spin_up_time - spin_up_time + stagger_wait_interval = max_spin_up_time - job.get_spin_up_time() assert stagger_wait_interval >= 0 request = V0InitialJobRequest( diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_miner_driver.py b/validator/app/src/compute_horde_validator/validator/tests/test_miner_driver.py index fa17550a3..f92c7cb47 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_miner_driver.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_miner_driver.py @@ -11,6 +11,7 @@ V0JobFinishedRequest, ) from compute_horde.mv_protocol.validator_requests import ( + V0JobAcceptedReceiptRequest, V0JobFinishedReceiptRequest, ) @@ -36,7 +37,7 @@ "expected_job_status_updates", "organic_job_status", "dummy_job_factory", - "expected_job_started_receipt", + "expected_job_accepted_receipt", "expected_job_finished_receipt", ), [ @@ -61,7 +62,7 @@ ["accepted", "failed"], OrganicJob.Status.FAILED, get_dummy_job_request_v0, - False, + True, False, ), ( @@ -103,7 +104,7 @@ async def test_miner_driver( expected_job_status_updates, organic_job_status, dummy_job_factory, - expected_job_started_receipt, + expected_job_accepted_receipt, expected_job_finished_receipt, ): miner, _ = await Miner.objects.aget_or_create(hotkey="miner_client") @@ -162,8 +163,7 @@ async def track_job_status_updates(x): def condition(_): return True - # FIXME - # if expected_job_started_receipt: - # assert miner_client._query_sent_models(condition, V0JobStartedReceiptRequest) + if expected_job_accepted_receipt: + assert miner_client._query_sent_models(condition, V0JobAcceptedReceiptRequest) if expected_job_finished_receipt: assert miner_client._query_sent_models(condition, V0JobFinishedReceiptRequest) From 697a7f66a8d210e08dddfe0c6b310af4b8f5e92f Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 17:56:56 +0600 Subject: [PATCH 32/40] Removed unused code --- compute_horde/compute_horde/utils.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/compute_horde/compute_horde/utils.py b/compute_horde/compute_horde/utils.py index 1be8f2a8d..84e00fa9e 100644 --- a/compute_horde/compute_horde/utils.py +++ b/compute_horde/compute_horde/utils.py @@ -3,7 +3,6 @@ import bittensor import pydantic -from pydantic import BeforeValidator from substrateinterface.exceptions import SubstrateRequestException if TYPE_CHECKING: @@ -65,16 +64,3 @@ def time_left(self): if self.timeout is None: raise ValueError("timeout was not specified") return self.timeout - self.passed_time() - - -def _empty_string_none(value: Any) -> Any: - """ - Converts value to None if it is empty-string, otherwise returns the same value. - Intended to be used with pydantic validators. - """ - if value == "": - return None - return value - - -empty_string_none = BeforeValidator(_empty_string_none) From a30f3196eeadc0339b60434e68b6d9a87c44a38d Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 20:12:08 +0600 Subject: [PATCH 33/40] Fix migration order of lib --- ...> 0003_remove_jobstartedreceipt_time_accepted_and_more.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename compute_horde/compute_horde/receipts/migrations/{0002_remove_jobstartedreceipt_time_accepted_and_more.py => 0003_remove_jobstartedreceipt_time_accepted_and_more.py} (95%) diff --git a/compute_horde/compute_horde/receipts/migrations/0002_remove_jobstartedreceipt_time_accepted_and_more.py b/compute_horde/compute_horde/receipts/migrations/0003_remove_jobstartedreceipt_time_accepted_and_more.py similarity index 95% rename from compute_horde/compute_horde/receipts/migrations/0002_remove_jobstartedreceipt_time_accepted_and_more.py rename to compute_horde/compute_horde/receipts/migrations/0003_remove_jobstartedreceipt_time_accepted_and_more.py index 232c72793..903d6ebbf 100644 --- a/compute_horde/compute_horde/receipts/migrations/0002_remove_jobstartedreceipt_time_accepted_and_more.py +++ b/compute_horde/compute_horde/receipts/migrations/0003_remove_jobstartedreceipt_time_accepted_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1.1 on 2024-10-18 19:07 +# Generated by Django 5.1.1 on 2024-10-23 14:11 import datetime @@ -7,7 +7,7 @@ class Migration(migrations.Migration): dependencies = [ - ("receipts", "0001_initial"), + ("receipts", "0002_jobstartedreceipt_is_organic"), ] operations = [ From f4b9ecb3bd6e07050d3dd68d711155faf93c63a4 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Thu, 24 Oct 2024 16:10:56 +0600 Subject: [PATCH 34/40] Fix receipts missing fields --- miner/app/src/compute_horde_miner/miner/liveness_check.py | 1 + .../miner/miner_consumer/validator_interface.py | 3 ++- miner/app/src/compute_horde_miner/miner/tasks.py | 6 ++++++ .../miner/tests/integration/test_mocked_executor_manager.py | 1 + .../app/src/compute_horde_validator/validator/tasks.py | 1 + 5 files changed, 11 insertions(+), 1 deletion(-) diff --git a/miner/app/src/compute_horde_miner/miner/liveness_check.py b/miner/app/src/compute_horde_miner/miner/liveness_check.py index c400a76a2..f372ff305 100644 --- a/miner/app/src/compute_horde_miner/miner/liveness_check.py +++ b/miner/app/src/compute_horde_miner/miner/liveness_check.py @@ -158,6 +158,7 @@ async def drive_executor() -> float: timestamp=datetime.datetime.now(datetime.UTC), executor_class=DEFAULT_EXECUTOR_CLASS, max_timeout=JOB_TIMEOUT_SECONDS, + is_organic=False, ttl=30, ) receipt_signature = f"0x{keypair.sign(receipt_payload.blob_for_signing()).hex()}" diff --git a/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py b/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py index 89aedb54b..d57d6cbe3 100644 --- a/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py +++ b/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py @@ -400,7 +400,7 @@ async def handle_job_started_receipt(self, payload: JobStartedReceiptPayload, si timestamp=payload.timestamp, executor_class=payload.executor_class, max_timeout=payload.max_timeout, - is_organic=msg.payload.is_organic, + is_organic=payload.is_organic, ttl=payload.ttl, ) @@ -422,6 +422,7 @@ async def handle_job_accepted_receipt( miner_hotkey=msg.payload.miner_hotkey, validator_hotkey=msg.payload.validator_hotkey, timestamp=msg.payload.timestamp, + time_accepted=msg.payload.time_accepted, ttl=msg.payload.ttl, ) prepare_receipts.delay() diff --git a/miner/app/src/compute_horde_miner/miner/tasks.py b/miner/app/src/compute_horde_miner/miner/tasks.py index 79f1a698e..953d52fef 100644 --- a/miner/app/src/compute_horde_miner/miner/tasks.py +++ b/miner/app/src/compute_horde_miner/miner/tasks.py @@ -119,6 +119,8 @@ def get_receipts_from_old_miner(): job_uuid=receipt.payload.job_uuid, miner_hotkey=receipt.payload.miner_hotkey, validator_hotkey=receipt.payload.validator_hotkey, + validator_signature=receipt.validator_signature, + miner_signature=receipt.miner_signature, timestamp=receipt.payload.timestamp, executor_class=receipt.payload.executor_class, max_timeout=receipt.payload.max_timeout, @@ -146,6 +148,8 @@ def get_receipts_from_old_miner(): job_uuid=receipt.payload.job_uuid, miner_hotkey=receipt.payload.miner_hotkey, validator_hotkey=receipt.payload.validator_hotkey, + validator_signature=receipt.validator_signature, + miner_signature=receipt.miner_signature, timestamp=receipt.payload.timestamp, time_accepted=receipt.payload.time_accepted, ttl=receipt.payload.ttl, @@ -173,6 +177,8 @@ def get_receipts_from_old_miner(): job_uuid=receipt.payload.job_uuid, miner_hotkey=receipt.payload.miner_hotkey, validator_hotkey=receipt.payload.validator_hotkey, + validator_signature=receipt.validator_signature, + miner_signature=receipt.miner_signature, timestamp=receipt.payload.timestamp, time_started=receipt.payload.time_started, time_took_us=receipt.payload.time_took_us, diff --git a/miner/app/src/compute_horde_miner/miner/tests/integration/test_mocked_executor_manager.py b/miner/app/src/compute_horde_miner/miner/tests/integration/test_mocked_executor_manager.py index 311da15ab..ff7646e8d 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/integration/test_mocked_executor_manager.py +++ b/miner/app/src/compute_horde_miner/miner/tests/integration/test_mocked_executor_manager.py @@ -100,6 +100,7 @@ async def run_regular_flow_test(validator_key: str, job_uuid: str): "timestamp": "2020-01-01T00:00Z", "executor_class": DEFAULT_EXECUTOR_CLASS, "max_timeout": 60, + "is_organic": True, "ttl": 5, }, "job_started_receipt_signature": "gibberish", diff --git a/validator/app/src/compute_horde_validator/validator/tasks.py b/validator/app/src/compute_horde_validator/validator/tasks.py index d1bc87804..3cb47b89e 100644 --- a/validator/app/src/compute_horde_validator/validator/tasks.py +++ b/validator/app/src/compute_horde_validator/validator/tasks.py @@ -1066,6 +1066,7 @@ def fetch_receipts_from_miner(hotkey: str, ip: str, port: int): miner_signature=receipt.miner_signature, validator_signature=receipt.validator_signature, timestamp=receipt.payload.timestamp, + time_accepted=receipt.payload.time_accepted, ttl=receipt.payload.ttl, ) for receipt in receipts From fcc5f4a755e36e3c74f4c4c6273926fa2298606b Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Thu, 24 Oct 2024 16:15:17 +0600 Subject: [PATCH 35/40] Fix miner receipts test --- .../miner/tests/test_receipts.py | 47 +++++++++++++------ 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py b/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py index 6376b6fce..b6870ef54 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py +++ b/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py @@ -9,14 +9,16 @@ JobFinishedReceiptPayload, JobStartedReceiptPayload, V0AuthenticateRequest, + V0InitialJobRequest, + V0JobAcceptedReceiptRequest, V0JobFinishedReceiptRequest, - V0JobStartedReceiptRequest, ) from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.schemas import JobAcceptedReceiptPayload from django.utils import timezone from pytest_mock import MockerFixture -from compute_horde_miner.miner.models import AcceptedJob, Validator +from compute_horde_miner.miner.models import Validator from compute_horde_miner.miner.tests.validator import fake_validator @@ -41,7 +43,7 @@ async def test_receipt_is_saved( settings.DEBUG_TURN_AUTHENTICATION_OFF = True settings.BITTENSOR_WALLET = lambda: miner_wallet job_uuid = str(uuid4()) - validator = await Validator.objects.acreate( + await Validator.objects.acreate( public_key=validator_wallet.hotkey.ss58_address, active=True, ) @@ -69,28 +71,44 @@ async def test_receipt_is_saved( }, } - # Skip doing the job - await AcceptedJob.objects.acreate( - job_uuid=job_uuid, - validator=validator, - initial_job_details={}, - ) - # Send the receipts job_started_receipt_payload = JobStartedReceiptPayload( job_uuid=job_uuid, is_organic=organic_job, miner_hotkey=miner_wallet.hotkey.ss58_address, validator_hotkey=validator_wallet.hotkey.ss58_address, + timestamp=timezone.now(), executor_class=ExecutorClass.spin_up_4min__gpu_24gb, + max_timeout=60, + ttl=5, + ) + job_started_receipt_signature = _sign(job_started_receipt_payload, validator_wallet.hotkey) + await fake_validator_channel.send_to( + V0InitialJobRequest( + job_uuid=job_uuid, + executor_class=ExecutorClass.spin_up_4min__gpu_24gb, + base_docker_image_name="it's teeeeests", + timeout_seconds=123, + job_started_receipt_payload=job_started_receipt_payload, + job_started_receipt_signature=job_started_receipt_signature, + ).model_dump_json() + ) + + # skip doing the job, and only send receipts + + job_accepted_receipt_payload = JobAcceptedReceiptPayload( + job_uuid=job_uuid, + miner_hotkey=miner_wallet.hotkey.ss58_address, + validator_hotkey=validator_wallet.hotkey.ss58_address, + timestamp=timezone.now(), time_accepted=timezone.now(), - max_timeout=123, + ttl=5, ) await fake_validator_channel.send_to( - V0JobStartedReceiptRequest( + V0JobAcceptedReceiptRequest( job_uuid=job_uuid, - payload=job_started_receipt_payload, - signature=_sign(job_started_receipt_payload, validator_wallet.hotkey), + payload=job_accepted_receipt_payload, + signature=_sign(job_accepted_receipt_payload, validator_wallet.hotkey), ).model_dump_json() ) @@ -98,6 +116,7 @@ async def test_receipt_is_saved( job_uuid=job_uuid, miner_hotkey=miner_wallet.hotkey.ss58_address, validator_hotkey=validator_wallet.hotkey.ss58_address, + timestamp=timezone.now(), time_started=timezone.now(), time_took_us=123, score_str="123.45", From fa596324e59f7d538a0adf5807186468d6bbcfa8 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Thu, 24 Oct 2024 17:37:44 +0600 Subject: [PATCH 36/40] Align receipts test with stub executor manager --- .../miner/tests/test_receipts.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py b/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py index b6870ef54..4f63e253f 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py +++ b/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py @@ -3,6 +3,7 @@ import bittensor import pytest +from compute_horde.base.volume import VolumeType from compute_horde.executor_class import ExecutorClass from compute_horde.mv_protocol.validator_requests import ( AuthenticationPayload, @@ -19,6 +20,7 @@ from pytest_mock import MockerFixture from compute_horde_miner.miner.models import Validator +from compute_horde_miner.miner.tests.executor_manager import fake_executor from compute_horde_miner.miner.tests.validator import fake_validator @@ -26,6 +28,14 @@ def _sign(msg, key: bittensor.Keypair): return f"0x{key.sign(msg.blob_for_signing()).hex()}" +@pytest.fixture +def job_uuid(): + _job_uuid = str(uuid4()) + fake_executor.job_uuid = _job_uuid + yield _job_uuid + fake_executor.job_uuid = None + + @pytest.mark.parametrize( "organic_job", (True, False), @@ -37,12 +47,12 @@ async def test_receipt_is_saved( miner_wallet: bittensor.wallet, mocker: MockerFixture, organic_job: bool, + job_uuid: str, settings, ) -> None: mocker.patch("compute_horde_miner.miner.miner_consumer.validator_interface.prepare_receipts") settings.DEBUG_TURN_AUTHENTICATION_OFF = True settings.BITTENSOR_WALLET = lambda: miner_wallet - job_uuid = str(uuid4()) await Validator.objects.acreate( public_key=validator_wallet.hotkey.ss58_address, active=True, @@ -88,7 +98,8 @@ async def test_receipt_is_saved( job_uuid=job_uuid, executor_class=ExecutorClass.spin_up_4min__gpu_24gb, base_docker_image_name="it's teeeeests", - timeout_seconds=123, + timeout_seconds=60, + volume_type=VolumeType.inline, job_started_receipt_payload=job_started_receipt_payload, job_started_receipt_signature=job_started_receipt_signature, ).model_dump_json() From e56d38fdffdc8eeb475e290095eb8e2f18249b7a Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Fri, 25 Oct 2024 01:02:23 +0600 Subject: [PATCH 37/40] Fix integration tests --- tests/integration_tests/test_miner_on_dev_executor_manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration_tests/test_miner_on_dev_executor_manager.py b/tests/integration_tests/test_miner_on_dev_executor_manager.py index ec8c1d852..636e574e1 100644 --- a/tests/integration_tests/test_miner_on_dev_executor_manager.py +++ b/tests/integration_tests/test_miner_on_dev_executor_manager.py @@ -161,6 +161,7 @@ async def test_echo_image(self): "timestamp": datetime.now(tz=UTC).isoformat(), "executor_class": DEFAULT_EXECUTOR_CLASS, "max_timeout": 60, + "is_organic": True, "ttl": 30, } blob = json.dumps(receipt_payload, sort_keys=True) From f749d4bd5f23515139a6b029b99c53207821f37f Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Mon, 28 Oct 2024 18:39:48 +0600 Subject: [PATCH 38/40] Add missing assert for miner receipts test --- miner/app/src/compute_horde_miner/miner/tests/test_receipts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py b/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py index 4f63e253f..b5b3bac63 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py +++ b/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py @@ -14,7 +14,7 @@ V0JobAcceptedReceiptRequest, V0JobFinishedReceiptRequest, ) -from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.models import JobAcceptedReceipt, JobFinishedReceipt, JobStartedReceipt from compute_horde.receipts.schemas import JobAcceptedReceiptPayload from django.utils import timezone from pytest_mock import MockerFixture @@ -143,4 +143,5 @@ async def test_receipt_is_saved( assert await JobStartedReceipt.objects.filter( job_uuid=job_uuid, is_organic=organic_job ).aexists() + assert await JobAcceptedReceipt.objects.filter(job_uuid=job_uuid).aexists() assert await JobFinishedReceipt.objects.filter(job_uuid=job_uuid).aexists() From 65f144b4892fc1e516730cda5a882785041050e8 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Mon, 28 Oct 2024 18:45:55 +0600 Subject: [PATCH 39/40] Use different seed for miner/validator wallets in tests --- tests/integration_tests/test_miner_on_dev_executor_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/test_miner_on_dev_executor_manager.py b/tests/integration_tests/test_miner_on_dev_executor_manager.py index 636e574e1..2610858e2 100644 --- a/tests/integration_tests/test_miner_on_dev_executor_manager.py +++ b/tests/integration_tests/test_miner_on_dev_executor_manager.py @@ -42,8 +42,8 @@ def get_validator_wallet(): wallet = bittensor.wallet(name="test_validator") try: # workaround the overwrite flag - wallet.regenerate_coldkey(seed="0" * 64, use_password=False, overwrite=True) - wallet.regenerate_hotkey(seed="1" * 64, use_password=False, overwrite=True) + wallet.regenerate_coldkey(seed="2" * 64, use_password=False, overwrite=True) + wallet.regenerate_hotkey(seed="3" * 64, use_password=False, overwrite=True) except Exception as e: logger.error(f"Failed to create wallet: {e}") return wallet From be1f0b888d89a13a656f11616865ae47da2be483 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Mon, 28 Oct 2024 18:56:24 +0600 Subject: [PATCH 40/40] Add indices for `timestamp` fields of receipt models --- ...py => 0003_jobacceptedreceipt_and_more.py} | 67 +++++++++++-------- .../compute_horde/receipts/models.py | 3 + 2 files changed, 43 insertions(+), 27 deletions(-) rename compute_horde/compute_horde/receipts/migrations/{0003_remove_jobstartedreceipt_time_accepted_and_more.py => 0003_jobacceptedreceipt_and_more.py} (71%) diff --git a/compute_horde/compute_horde/receipts/migrations/0003_remove_jobstartedreceipt_time_accepted_and_more.py b/compute_horde/compute_horde/receipts/migrations/0003_jobacceptedreceipt_and_more.py similarity index 71% rename from compute_horde/compute_horde/receipts/migrations/0003_remove_jobstartedreceipt_time_accepted_and_more.py rename to compute_horde/compute_horde/receipts/migrations/0003_jobacceptedreceipt_and_more.py index 903d6ebbf..1ec5d3df6 100644 --- a/compute_horde/compute_horde/receipts/migrations/0003_remove_jobstartedreceipt_time_accepted_and_more.py +++ b/compute_horde/compute_horde/receipts/migrations/0003_jobacceptedreceipt_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1.1 on 2024-10-23 14:11 +# Generated by Django 5.1.1 on 2024-10-28 12:53 import datetime @@ -11,6 +11,28 @@ class Migration(migrations.Migration): ] operations = [ + migrations.CreateModel( + name="JobAcceptedReceipt", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("job_uuid", models.UUIDField()), + ("validator_hotkey", models.CharField(max_length=256)), + ("miner_hotkey", models.CharField(max_length=256)), + ("validator_signature", models.CharField(max_length=256)), + ("miner_signature", models.CharField(blank=True, max_length=256, null=True)), + ("timestamp", models.DateTimeField()), + ("time_accepted", models.DateTimeField()), + ("ttl", models.IntegerField()), + ], + options={ + "abstract": False, + }, + ), migrations.RemoveField( model_name="jobstartedreceipt", name="time_accepted", @@ -37,31 +59,22 @@ class Migration(migrations.Migration): field=models.IntegerField(default=0), preserve_default=False, ), - migrations.CreateModel( - name="JobAcceptedReceipt", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, primary_key=True, serialize=False, verbose_name="ID" - ), - ), - ("job_uuid", models.UUIDField()), - ("validator_hotkey", models.CharField(max_length=256)), - ("miner_hotkey", models.CharField(max_length=256)), - ("validator_signature", models.CharField(max_length=256)), - ("miner_signature", models.CharField(blank=True, max_length=256, null=True)), - ("timestamp", models.DateTimeField()), - ("time_accepted", models.DateTimeField()), - ("ttl", models.IntegerField()), - ], - options={ - "abstract": False, - "constraints": [ - models.UniqueConstraint( - fields=("job_uuid",), name="receipts_unique_jobacceptedreceipt_job_uuid" - ) - ], - }, + migrations.AddIndex( + model_name="jobfinishedreceipt", + index=models.Index(fields=["timestamp"], name="jobfinishedreceipt_ts_idx"), + ), + migrations.AddIndex( + model_name="jobstartedreceipt", + index=models.Index(fields=["timestamp"], name="jobstartedreceipt_ts_idx"), + ), + migrations.AddIndex( + model_name="jobacceptedreceipt", + index=models.Index(fields=["timestamp"], name="jobacceptedreceipt_ts_idx"), + ), + migrations.AddConstraint( + model_name="jobacceptedreceipt", + constraint=models.UniqueConstraint( + fields=("job_uuid",), name="receipts_unique_jobacceptedreceipt_job_uuid" + ), ), ] diff --git a/compute_horde/compute_horde/receipts/models.py b/compute_horde/compute_horde/receipts/models.py index 56a3a27e5..04c45f662 100644 --- a/compute_horde/compute_horde/receipts/models.py +++ b/compute_horde/compute_horde/receipts/models.py @@ -28,6 +28,9 @@ class Meta: constraints = [ models.UniqueConstraint(fields=["job_uuid"], name="receipts_unique_%(class)s_job_uuid"), ] + indexes = [ + models.Index(fields=["timestamp"], name="%(class)s_ts_idx"), + ] def __str__(self): return f"job_uuid: {self.job_uuid}"