Merge branch 'master' into prioritize-storing-jobs

.github/workflows/library_ci.yml

@@ -28,6 +28,8 @@ jobs:
         run: python -m pip install --upgrade nox 'pdm>=2.12,<3'
       - name: Run linters
         run: nox -vs lint
+      - name: Check for missing migrations
+        run: nox -vs check_missing_migrations
   type_check:
     runs-on: ubuntu-latest
     defaults:

.github/workflows/vulnerability_scan.yml

@@ -0,0 +1,43 @@
+name: "Vulnerability Scan"
+
+on:
+  schedule:
+    - cron: "0 0 * * *"
+
+env:
+  # temporary workaround for trivy db update issue
+  # https://github.com/aquasecurity/trivy/discussions/7538#discussioncomment-10741936
+  TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2
+
+jobs:
+  trivy:
+    strategy:
+      matrix:
+        image_name: [
+          "backenddevelopersltd/compute-horde-miner:v0-latest",
+          "backenddevelopersltd/compute-horde-miner-runner:v0-latest",
+          "backenddevelopersltd/compute-horde-miner-nginx:v0-latest",
+          "backenddevelopersltd/compute-horde-validator:v0-latest",
+          "backenddevelopersltd/compute-horde-validator-runner:v0-latest",
+          "backenddevelopersltd/compute-horde-validator-nginx:v0-latest",
+          "backenddevelopersltd/compute-horde-executor:v0-latest",
+          "backenddevelopersltd/compute-horde-job:v0-latest",
+          "backenddevelopersltd/compute-horde-job:v1-latest",
+        ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@0.24.0
+        with:
+          image-ref: "${{ matrix.image_name }}"
+          scanners: "vuln"
+          severity: 'CRITICAL,HIGH,MEDIUM'
+          limit-severities-for-sarif: true
+          ignore-unfixed: true
+          format: 'sarif'
+          output: 'trivy-results.sarif'
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v3
+        with:
+          sarif_file: 'trivy-results.sarif'

.gitignore

@@ -4,13 +4,18 @@
 *.egg-info/
 .idea/
 .env
-venv
+venv/
+.venv/
 .hypothesis
 .envrc
 .nox/
-__pycache__
-build
-dist
+.mypy_cache/
+.pdm-build/
+.pytest_cache/
+.ruff_cache/
+__pycache__/
+build/
+dist/
 .pdm-python
 /wallets/
 /facilitator/

compute_horde/README.md

@@ -1 +1,18 @@
-# compute-horde
+# compute-horde
+
+## Common django models
+This library contains common Django models for the receipts.
+To use them, update your `INSTALLED_APPS`:
+```python
+INSTALLED_APPS = [
+    ...,
+    'compute_horde.receipts',
+    ...,
+]
+```
+
+## Migrations
+To make new migrations after doing some changes in the model files, run:
+```shell
+DJANGO_SETTINGS_MODULE=compute_horde.settings pdm run django-admin makemigrations
+```

compute_horde/compute_horde/base/admin.py

@@ -0,0 +1,17 @@
+class AddOnlyAdminMixin:
+    def has_change_permission(self, *args, **kwargs):
+        return False
+
+    def has_delete_permission(self, *args, **kwargs):
+        return False
+
+
+class ReadOnlyAdminMixin:
+    def has_change_permission(self, *args, **kwargs):
+        return False
+
+    def has_delete_permission(self, *args, **kwargs):
+        return False
+
+    def has_add_permission(self, *args, **kwargs):
+        return False

compute_horde/compute_horde/base/volume.py

@@ -14,11 +14,22 @@ class VolumeType(str, enum.Enum):
     zip_url = "zip_url"
     single_file = "single_file"
     multi_volume = "multi_volume"
+    huggingface_volume = "huggingface_volume"
 
     def __str__(self):
         return str.__str__(self)
 
 
+class HuggingfaceVolume(pydantic.BaseModel):
+    volume_type: Literal[VolumeType.huggingface_volume] = VolumeType.huggingface_volume
+    repo_id: str
+    revision: str | None = None  # Git revision id: branch name / tag / commit hash
+    relative_path: str | None = None
+
+    def is_safe(self) -> bool:
+        return True
+
+
 class InlineVolume(pydantic.BaseModel):
     volume_type: Literal[VolumeType.inline] = VolumeType.inline
     contents: str
@@ -56,7 +67,8 @@ class MultiVolume(pydantic.BaseModel):
     volume_type: Literal[VolumeType.multi_volume] = VolumeType.multi_volume
     volumes: list[
         Annotated[
-            InlineVolume | ZipUrlVolume | SingleFileVolume, Field(discriminator="volume_type")
+            InlineVolume | ZipUrlVolume | SingleFileVolume | HuggingfaceVolume,
+            Field(discriminator="volume_type"),
         ]
     ]
 
@@ -65,6 +77,6 @@ def is_safe(self) -> bool:
 
 
 Volume = Annotated[
-    InlineVolume | ZipUrlVolume | SingleFileVolume | MultiVolume,
+    InlineVolume | ZipUrlVolume | SingleFileVolume | MultiVolume | HuggingfaceVolume,
     Field(discriminator="volume_type"),
 ]

compute_horde/compute_horde/em_protocol/miner_requests.py

@@ -3,6 +3,7 @@
 
 from pydantic import model_validator
 
+from ..base.docker import DockerRunOptionsPreset
 from ..base.output_upload import OutputUpload, OutputUploadType  # noqa
 from ..base.volume import Volume, VolumeType
 from ..base_requests import BaseRequest, JobMixin
@@ -36,7 +37,7 @@ class V0JobRequest(BaseMinerRequest, JobMixin):
     message_type: RequestType = RequestType.V0RunJobRequest
     docker_image_name: str | None = None
     raw_script: str | None = None
-    docker_run_options_preset: str
+    docker_run_options_preset: DockerRunOptionsPreset
     docker_run_cmd: list[str]
     volume: Volume | None = None
     output_upload: OutputUpload | None = None

compute_horde/compute_horde/fv_protocol/facilitator_requests.py

@@ -0,0 +1,139 @@
+from typing import Annotated, Literal, Self
+
+import pydantic
+from pydantic import BaseModel, JsonValue, model_validator
+
+from compute_horde.base.output_upload import OutputUpload, ZipAndHttpPutUpload
+from compute_horde.base.volume import Volume, ZipUrlVolume
+from compute_horde.executor_class import ExecutorClass
+
+
+class Error(BaseModel, extra="allow"):
+    msg: str
+    type: str
+    help: str = ""
+
+
+class Response(BaseModel, extra="forbid"):
+    """Message sent from facilitator to validator in response to AuthenticationRequest & JobStatusUpdate"""
+
+    status: Literal["error", "success"]
+    errors: list[Error] = []
+
+
+class Signature(BaseModel, extra="forbid"):
+    # has defaults to allow easy instantiation
+    signature_type: str = ""
+    signatory: str = ""
+    timestamp_ns: int = 0
+    signature: str = ""
+
+
+class V0JobRequest(BaseModel, extra="forbid"):
+    """Message sent from facilitator to validator to request a job execution"""
+
+    # this points to a `ValidatorConsumer.job_new` handler (fuck you django-channels!)
+    type: Literal["job.new"] = "job.new"
+    message_type: Literal["V0JobRequest"] = "V0JobRequest"
+
+    uuid: str
+    miner_hotkey: str
+    executor_class: ExecutorClass
+    docker_image: str
+    raw_script: str
+    args: list[str]
+    env: dict[str, str]
+    use_gpu: bool
+    input_url: str
+    output_url: str
+
+    def get_args(self):
+        return self.args
+
+    @model_validator(mode="after")
+    def validate_at_least_docker_image_or_raw_script(self) -> Self:
+        if not (bool(self.docker_image) or bool(self.raw_script)):
+            raise ValueError("Expected at least one of `docker_image` or `raw_script`")
+        return self
+
+    @property
+    def volume(self) -> Volume | None:
+        if self.input_url:
+            return ZipUrlVolume(contents=self.input_url)
+        return None
+
+    @property
+    def output_upload(self) -> OutputUpload | None:
+        if self.output_url:
+            return ZipAndHttpPutUpload(url=self.output_url)
+        return None
+
+
+class V1JobRequest(BaseModel, extra="forbid"):
+    """Message sent from facilitator to validator to request a job execution"""
+
+    # this points to a `ValidatorConsumer.job_new` handler (fuck you django-channels!)
+    type: Literal["job.new"] = "job.new"
+    message_type: Literal["V1JobRequest"] = "V1JobRequest"
+    uuid: str
+    miner_hotkey: str
+    executor_class: ExecutorClass
+    docker_image: str
+    raw_script: str
+    args: list[str]
+    env: dict[str, str]
+    use_gpu: bool
+    volume: Volume | None = None
+    output_upload: OutputUpload | None = None
+
+    def get_args(self):
+        return self.args
+
+    @model_validator(mode="after")
+    def validate_at_least_docker_image_or_raw_script(self) -> Self:
+        if not (bool(self.docker_image) or bool(self.raw_script)):
+            raise ValueError("Expected at least one of `docker_image` or `raw_script`")
+        return self
+
+
+class V2JobRequest(BaseModel, extra="forbid"):
+    """Message sent from facilitator to validator to request a job execution"""
+
+    # this points to a `ValidatorConsumer.job_new` handler (fuck you django-channels!)
+    type: Literal["job.new"] = "job.new"
+    message_type: Literal["V2JobRequest"] = "V2JobRequest"
+    signature: Signature | None = None
+
+    # !!! all fields below are included in the signed json payload
+    uuid: str
+    executor_class: ExecutorClass
+    docker_image: str
+    raw_script: str
+    args: list[str]
+    env: dict[str, str]
+    use_gpu: bool
+    volume: Volume | None = None
+    output_upload: OutputUpload | None = None
+    # !!! all fields above are included in the signed json payload
+
+    def get_args(self):
+        return self.args
+
+    def json_for_signing(self) -> JsonValue:
+        payload = self.model_dump(mode="json")
+        del payload["type"]
+        del payload["message_type"]
+        del payload["signature"]
+        return payload
+
+    @model_validator(mode="after")
+    def validate_at_least_docker_image_or_raw_script(self) -> Self:
+        if not (bool(self.docker_image) or bool(self.raw_script)):
+            raise ValueError("Expected at least one of `docker_image` or `raw_script`")
+        return self
+
+
+JobRequest = Annotated[
+    V0JobRequest | V1JobRequest | V2JobRequest,
+    pydantic.Field(discriminator="message_type"),
+]

compute_horde/compute_horde/fv_protocol/validator_requests.py

@@ -0,0 +1,50 @@
+from typing import Any, Literal, Self
+
+import bittensor
+from pydantic import BaseModel
+
+
+class V0Heartbeat(BaseModel, extra="forbid"):
+    """Message sent from validator to facilitator to keep connection alive"""
+
+    message_type: Literal["V0Heartbeat"] = "V0Heartbeat"
+
+
+class V0AuthenticationRequest(BaseModel, extra="forbid"):
+    """Message sent from validator to facilitator to authenticate itself"""
+
+    message_type: Literal["V0AuthenticationRequest"] = "V0AuthenticationRequest"
+    public_key: str
+    signature: str
+
+    @classmethod
+    def from_keypair(cls, keypair: bittensor.Keypair) -> Self:
+        return cls(
+            public_key=keypair.public_key.hex(),
+            signature=f"0x{keypair.sign(keypair.public_key).hex()}",
+        )
+
+    def verify_signature(self) -> bool:
+        public_key_bytes = bytes.fromhex(self.public_key)
+        keypair = bittensor.Keypair(public_key=public_key_bytes, ss58_format=42)
+        # make mypy happy
+        valid: bool = keypair.verify(public_key_bytes, self.signature)
+        return valid
+
+    @property
+    def ss58_address(self) -> str:
+        # make mypy happy
+        address: str = bittensor.Keypair(
+            public_key=bytes.fromhex(self.public_key), ss58_format=42
+        ).ss58_address
+        return address
+
+
+class V0MachineSpecsUpdate(BaseModel, extra="forbid"):
+    """Message sent from validator to facilitator to update miner specs"""
+
+    message_type: Literal["V0MachineSpecsUpdate"] = "V0MachineSpecsUpdate"
+    miner_hotkey: str
+    validator_hotkey: str
+    specs: dict[str, Any]
+    batch_id: str