diff --git a/.gitignore b/.gitignore index 68bc17f..1c3f414 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,84 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# SonarLint plugin +.idea/sonarlint/ + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +# Validators db. +database.db diff --git a/README.md b/README.md index b3d06fb..72bb674 100644 --- a/README.md +++ b/README.md @@ -1,59 +1,136 @@ +# Compute Subnet -
- -# **Bittensor Compute Subnet** [![Discord Chat](https://img.shields.io/discord/308323056592486420.svg)](https://discord.gg/bittensor) -[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) --- -### The Incentivized Internet +### The Incentivized Internet [Discord](https://discord.gg/bittensor) • [Network](https://taostats.io/) • [Research](https://bittensor.com/whitepaper) -
+This repository contains all the necessary files and functions to define Bittensor's Compute Subnet. It enables running miners on netuid 15 in Bittensor's test network or netuid 27 in Bittensor's main network. ---- +## Introduction -This repo contains all the necessary files and functions to define Bittensor's Compute Subnet. You can try running miners on netuid 15 in Bittensor's test network. +This repository serves as a compute-composable subnet, integrating various cloud platforms (e.g., Runpod, Lambda, AWS) into a cohesive unit. Its purpose is to enable higher-level cloud platforms to offer seamless compute composability across different underlying platforms. With the proliferation of cloud platforms, there's a growing need for a subnet that can seamlessly integrate these platforms, allowing efficient resource sharing and allocation. This compute-composable subnet empowers nodes to contribute computational power, with validators ensuring the integrity and efficiency of the shared resources. -# Introduction -This repository is a compute-composable subnet. This subnet has integrated various cloud platforms (e.g., Runpod, Lambda, AWS) into a cohesive unit, enabling higher-level cloud platforms to offer seamless compute composability across different underlying platforms. With the proliferation of cloud platforms, there's a need for a subnet that can seamlessly integrate these platforms, allowing for efficient resource sharing and allocation. This compute-composable subnet will enable nodes to contribute computational power, with validators ensuring the integrity and efficiency of the shared resources. +### File Structure -- `compute/protocol.py`: The file where the wire-protocol used by miners and validators is defined. -- `neurons/miner.py`: This script which defines the miner's behavior, i.e., how the miner responds to requests from validators. -- `neurons/validator.py`: This script which defines the validator's behavior, i.e., how the validator requests information from miners and determines scores. +- `compute/protocol.py`: Defines the wire-protocol used by miners and validators. +- `neurons/miner.py`: Defines the miner's behavior in responding to requests from validators. +- `neurons/validator.py`: Defines the validator's behavior in requesting information from miners and determining scores. ---- +## Installation -# Installation This repository requires python3.8 or higher. To install, simply clone this repository and install the requirements. -## Install Bittensor +### Bittensor + ```bash -$ /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/opentensor/bittensor/master/scripts/install.sh)" +/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/opentensor/bittensor/master/scripts/install.sh)" ``` -## Install Dependencies + +## Dependencies - Validators / Miners + ```bash git clone https://github.com/neuralinternet/Compute-Subnet.git cd Compute-Subnet python3 -m pip install -r requirements.txt python3 -m pip install -e . ``` -## Setup Docker for Miner -To run a miner, you must [install](https://docs.docker.com/engine/install/ubuntu) and start the docker service by running `sudo systemctl start docker` and `sudo apt install at`. - +## Extra dependencies - Miners + +### Hashcat + +```bash +# Minimal hashcat version >= v6.2.6 +wget https://hashcat.net/files/hashcat-6.2.6.tar.gz +tar xzvf hashcat-6.2.6.tar.gz +cd hashcat-6.2.6/ +make +make install # prefixed by sudo if not in the sudoers +hashcat --version +``` + +### Cuda + +```bash +# Recommended cuda version: 12.3 +wget https://developer.download.nvidia.com/compute/cuda/12.3.1/local_installers/cuda-repo-ubuntu2204-12-3-local_12.3.1-545.23.08-1_amd64.deb +dpkg -i cuda-repo-ubuntu2204-12-3-local_12.3.1-545.23.08-1_amd64.deb +cp /var/cuda-repo-ubuntu2204-12-3-local/cuda-*-keyring.gpg /usr/share/keyrings/ +apt-get update +apt-get -y install cuda-toolkit-12-3 +apt-get -y install -y cuda-drivers + +# Valid for x64 architecture. Consult nvidia documentation for any other architecture. +export CUDA_VERSION=cuda-12.3 +export PATH=$PATH:/usr/local/$CUDA_VERSION/bin +export LD_LIBRARY_PATH=/usr/local/$CUDA_VERSION/lib64 + +echo "">>~/.bashrc +echo "PATH=$PATH">>~/.bashrc +echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH">>~/.bashrc + +reboot # Changes might need a restart depending on the system + +nvidia-smi +nvcc --version + +# Version should match +``` + +### Docker + +To run a miner, you must [install](https://docs.docker.com/engine/install/ubuntu) and start the docker service. + +```bash +sudo apt install docker.io -y +sudo apt install docker-compose -y +sudo systemctl start docker +sudo apt install at +docker run hello-world # Must not return you any error. +``` + +### Running subtensor locally + +```bash +git clone https://github.com/opentensor/subtensor.git +cd subtensor +docker-compose up --detach +``` + +If you have more complicated needs, see the [subtensor](https://github.com/opentensor/subtensor/) repo for more details and understanding. + --- + # Running a Miner / Validator -Prior to running a miner or validator, you must [create a wallet](https://github.com/opentensor/docs/blob/main/reference/btcli.md) and [register the wallet to a netuid](https://github.com/opentensor/docs/blob/main/subnetworks/registration.md). Once you have done so, you can run the miner and validator with the following commands. + +Prior to running a miner or validator, you must [create a wallet](https://github.com/opentensor/docs/blob/main/reference/btcli.md) +and [register the wallet to a netuid](https://github.com/opentensor/docs/blob/main/subnetworks/registration.md). +Once you have done so, you can run the miner and validator with the following commands. ## Running Miner -Miners contribute processing resources, notably GPU (Graphics Processing Unit) and CPU (Central Processing Unit) instances, to facilitate optimal performance in essential GPU and CPU-based computing tasks. The system operates on a performance-based reward mechanism, where miners are incentivized through a tiered reward structure correlated to the processing capability of their hardware. High-performance devices are eligible for increased compensation, reflecting their greater contribution to the network's computational throughput. Emphasizing the integration of GPU instances is critical due to their superior computational power, particularly in tasks demanding parallel processing capabilities. Consequently, miners utilizing GPU instances are positioned to receive substantially higher rewards compared to their CPU counterparts, in alignment with the greater processing power and efficiency GPUs bring to the network. +A dedicated medium article is available [here](https://medium.com/@neuralinternet/how-to-run-a-compute-miner-82498b93e7e1) + +Miners contribute processing resources, notably GPU (Graphics Processing Unit) and CPU (Central Processing Unit) +instances, to facilitate optimal performance in essential GPU and CPU-based computing tasks. The system operates on a +performance-based reward mechanism, where miners are incentivized through a tiered reward structure correlated to the +processing capability of their hardware. High-performance devices are eligible for increased compensation, reflecting +their greater contribution to the network's computational throughput. Emphasizing the integration of GPU instances is +critical due to their superior computational power, particularly in tasks demanding parallel processing capabilities. +Consequently, miners utilizing GPU instances are positioned to receive substantially higher rewards compared to their +CPU counterparts, in alignment with the greater processing power and efficiency GPUs bring to the network. -A key aspect of the miners' contribution is the management of resource reservations. Miners have the autonomy to set specific timelines for each reservation of their computational resources. This timeline dictates the duration for which the resources are allocated to a particular task or user. Once the set timeline reaches its conclusion, the reservation automatically expires, thereby freeing up the resources for subsequent allocations. This mechanism ensures a dynamic and efficient distribution of computational power, catering to varying demands within the network. +A key aspect of the miners' contribution is the management of resource reservations. Miners have the autonomy to set +specific timelines for each reservation of their computational resources. This timeline dictates the duration for which +the resources are allocated to a particular task or user. Once the set timeline reaches its conclusion, the reservation +automatically expires, thereby freeing up the resources for subsequent allocations. This mechanism ensures a dynamic and +efficient distribution of computational power, catering to varying demands within the network. ```bash # To run the miner @@ -68,104 +145,69 @@ python -m miner.py ## Running Validator -Validators hold the critical responsibility of rigorously assessing and verifying the computational capabilities of miners. This multifaceted evaluation process commences with validators requesting miners to provide comprehensive performance data, which includes not only processing speeds and efficiencies but also critical metrics like Random Access Memory (RAM) capacity and disk space availability. - -The inclusion of RAM and disk space measurements is vital, as these components significantly impact the overall performance and reliability of the miners' hardware. RAM capacity influences the ability to handle large or multiple tasks simultaneously, while adequate disk space ensures sufficient storage. - -Following the receipt of this detailed hardware and performance information, validators proceed to test the miners' computational integrity. This is achieved by presenting them with complex hashing challenges, designed to evaluate the processing power and reliability of the miners' systems. Validators adjust the difficulty of these problems based on the comprehensive performance profile of each miner, including their RAM and disk space metrics. - -In addition to measuring the time taken by miners to resolve these problems, validators meticulously verify the accuracy of the responses. This thorough examination of both speed and precision, complemented by the assessment of RAM and disk space utilization, forms the crux of the evaluation process. - -Based on this extensive analysis, validators update the miners' scores, reflecting a holistic view of their computational capacity, efficiency, and hardware quality. This score then determines the miner's weight within the network, directly influencing their potential rewards and standing. -This scoring process, implemented through a Python script, considers various factors including CPU, GPU, hard disk, and RAM performance. The script's structure and logic are outlined below: -1. **Score Calculation Function:** - - The `score` function aggregates performance data from different hardware components. - - It calculates individual scores for CPU, GPU, hard disk, and RAM. - - These scores are then weighted and summed to produce a total score. - - A registration bonus is applied if the miner is registered, enhancing the total score. - -2. **Component-Specific Scoring Functions:** - - `get_cpu_score`, `get_gpu_score`, `get_hard_disk_score`, and `get_ram_score` are functions dedicated to calculating scores for each respective hardware component. - - These functions consider the count, frequency, capacity, and speed of each component, applying a specific level value for normalization. - - The scores are derived based on the efficiency and capacity of the hardware. - -3. **Weight Assignment:** - - The script defines weights for each hardware component's score, signifying their importance in the overall performance. - - GPU has the highest weight (0.55), reflecting its significance in mining operations. - - CPU, hard disk, and RAM have lower weights (0.2, 0.1, and 0.15, respectively). - -4. **Registration Check:** - - The `check_if_registered` function verifies if a miner is registered using an external API (`wandb.Api()`). - - Registered miners receive a bonus to their total score, incentivizing official registration in the network. - -5. **Score Aggregation:** - - The individual scores are combined into a numpy array, `score_list`. - - The weights are also arranged in an array, `weight_list`. - - The final score is calculated using a dot product of these arrays, multiplied by 10, and then adjusted with the registration bonus. - -6. **Handling Multiple CPUs/GPUs:** - - The scoring functions for CPUs (`get_cpu_score`) and GPUs (`get_gpu_score`) are designed to process data that can represent multiple units. - - For CPUs, the `count` variable in `cpu_info` represents the number of CPU cores or units available. The score is calculated based on the cumulative capability, taking into account the total count and frequency of all CPU cores. - - For GPUs, similar logic applies. The script can handle data representing multiple GPUs, calculating a cumulative score based on their collective capacity and speed. - -7. **CPU Scoring (Multiple CPUs):** - - The CPU score is computed by multiplying the total number of CPU cores (`count`) by their average frequency (`frequency`), normalized against a predefined level. - - This approach ensures that configurations with multiple CPUs are appropriately rewarded for their increased processing power. - -8. **GPU Scoring (Multiple GPUs):** - - The GPU score is calculated by considering the total capacity (`capacity`) and the average speed (average of `graphics_speed` and `memory_speed`) of all GPUs in the system. - - The score reflects the aggregate performance capabilities of all GPUs, normalized against a set level. - - This method effectively captures the enhanced computational power provided by multiple GPU setups. - -9. **Aggregated Performance Assessment:** - - The final score calculation in the `score` function integrates the individual scores from CPU, GPU, hard disk, and RAM. - - This integration allows the scoring system to holistically assess the collective performance of all hardware components, including scenarios with multiple CPUs and GPUs. - -10. **Implications for Miners:** - - Miners with multiple GPUs and/or CPUs stand to gain a higher score due to the cumulative calculation of their hardware's capabilities. - - This approach incentivizes miners to enhance their hardware setup with additional CPUs and GPUs, thereby contributing more processing power to the network. - -The weight assignments are as follows: -- **GPU Weight:** 0.55 -- **CPU Weight:** 0.2 -- **Hard Disk Weight:** 0.1 -- **RAM Weight:** 0.15 +Validators hold the critical responsibility of rigorously assessing and verifying the computational capabilities of +miners. This multifaceted evaluation process commences with validators requesting miners to provide comprehensive +performance data, which includes not only processing speeds and efficiencies but also critical metrics like Random +Access Memory (RAM) capacity and disk space availability. -### Example 1: Miner A's Hardware Scores and Weighted Total +The inclusion of RAM and disk space measurements is vital, as these components significantly impact the overall +performance and reliability of the miners' hardware. RAM capacity influences the ability to handle large or multiple +tasks simultaneously, while adequate disk space ensures sufficient storage. -1. **CPU Score:** Calculated as `(2 cores * 3.0 GHz) / 1024 / 50`. -2. **GPU Score:** Calculated as `(8 GB * (1 GHz + 1 GHz) / 2) / 200000`. -3. **Hard Disk Score:** Calculated as `(500 GB * (100 MB/s + 100 MB/s) / 2) / 1000000`. -4. **RAM Score:** Calculated as `(16 GB * 2 GB/s) / 200000`. +Following the receipt of this detailed hardware and performance information, validators proceed to test the miners' +computational integrity. This is achieved by presenting them with complex hashing challenges, designed to evaluate the +processing power and reliability of the miners' systems. Validators adjust the difficulty of these problems based on the +comprehensive performance profile of each miner, including their RAM and disk space metrics. -Now, applying the weights: +In addition to measuring the time taken by miners to resolve these problems, validators meticulously verify the accuracy +of the responses. This thorough examination of both speed and precision, complemented by the assessment of RAM and disk +space utilization, forms the crux of the evaluation process. -- Total Score = (CPU Score × 0.2) + (GPU Score × 0.55) + (Hard Disk Score × 0.1) + (RAM Score × 0.15) -- If registered, add a registration bonus. +Based on this extensive analysis, validators update the miners' scores, reflecting a holistic view of their +computational capacity, efficiency, and hardware quality. This score then determines the miner's weight within the +network, directly influencing their potential rewards and standing. +This scoring process, implemented through a Python script, considers various factors including CPU, GPU, hard disk, and +RAM performance. The script's structure and logic are outlined below: -### Example 2: Miner B's Hardware Scores and Weighted Total +## Understanding the Score Calculation Process + +**The scoring system has been updated, if you want to check the old hardware mechanism:** [Hardware scoring](docs/hardware_scoring.md) + +The score calculation function determines a miner's performance based on various factors: -1. **CPU Score:** Calculated as `(4 cores * 2.5 GHz) / 1024 / 50`. -2. **GPU Score:** Calculated as `((6 GB + 6 GB) * (1.5 GHz + 1.2 GHz) / 2) / 200000`. -3. **Hard Disk Score:** Calculated as `(1 TB * (200 MB/s + 150 MB/s) / 2) / 1000000`. -4. **RAM Score:** Calculated as `(32 GB * 3 GB/s) / 200000`. +**Successful Problem Resolution**: It first checks if the problem was solved successfully. If not, the score remains at zero. -Applying the weights: +**Problem Difficulty**: This measures the complexity of the solved task. The code restricts this difficulty to a maximum allowed value. -- Total Score = (CPU Score × 0.2) + (GPU Score × 0.55) + (Hard Disk Score × 0.1) + (RAM Score × 0.15) -- Since Miner B is not registered, no registration bonus is added. +**Weighting Difficulty and Elapsed Time**: The function assigns a weight to both the difficulty of the solved problem (75%) and the time taken to solve it (25%). -### Impact of Weights on Total Score +**Exponential Rewards for Difficulty**: Higher problem difficulty leads to more significant rewards. An exponential formula is applied to increase rewards based on difficulty. -- The GPU has the highest weight (0.55), signifying its paramount importance in mining tasks, which are often GPU-intensive. Miners with powerful GPUs will thus receive a significantly higher portion of their total score from the GPU component. -- The CPU, while important, has a lower weight (0.2), reflecting its lesser but still vital role in the mining process. -- The hard disk and RAM have the lowest weights (0.1 and 0.15, respectively), indicating their supportive but less critical roles compared to GPU and CPU. +**Allocation Bonus**: Miners that have allocated machine receive an additional bonus added to their final score. -It is important to note that the role of validators, in contrast to miners, does not require the integration of GPU instances. Their function revolves around data integrity and accuracy verification, involving relatively modest network traffic and lower computational demands. As a result, their hardware requirements are less intensive, focusing more on stability and reliability rather than high-performance computation. +**Effect of Elapsed Time**: The time taken to solve the problem impacts the score. A shorter time results in a higher score. -### Resource Allocation -Validators can send requests to reserve access to resources from miners by specifying the specs manually in the in `register.py` and running this script: https://github.com/neuralinternet/Compute-Subnet/blob/main/neurons/register.py for example: -```{'cpu':{'count':1}, 'gpu':{}, 'hard_disk':{'capacity':10737418240}, 'ram':{'capacity':1073741824}}``` +- Max Score = 1e5 +- Score = Lowest Difficulty + (Difficulty Weight * Problem Difficulty) + (Elapsed Time * 1 / (1 + Elapsed Time) * 10000) + Allocation Bonus +- Normalized Score = (Score / Max Score) * 100 + +### Example 1: Miner A's Hardware Scores and Weighted Total + +- **Successful Problem Resolution**: True +- **Elapsed Time**: 4 seconds +- **Problem Difficulty**: 6 +- **Allocation**: True + +Score = 8.2865 + +### Example 2: Miner B's Hardware Scores and Weighted Total + +- **Successful Problem Resolution**: True +- **Elapsed Time**: 16 seconds +- **Problem Difficulty**: 8 +- **Allocation**: True + +Score = 24.835058823529412 ```bash # To run the validator @@ -178,12 +220,87 @@ python -m validator.py --logging.debug # Run in debug mode, alternatively --logging.trace for trace mode ``` - +## Resource Allocation Mechanism + +The allocation mechanism within subnet 27 is designed to optimize the utilization of computational resources +effectively. Key aspects of this mechanism include: + +1. **Resource Requirement Analysis:** The mechanism begins by analyzing the specific resource requirements of each task, + including CPU, GPU, memory, and storage needs. + +2. **Miner Selection:** Based on the analysis, the mechanism selects suitable miners that meet the resource + requirements. This selection process considers the current availability, performance history, and network weights of + the miners. + +3. **Dynamic Allocation:** The allocation of tasks to miners is dynamic, allowing for real-time adjustments based on + changing network conditions and miner performance. + +4. **Efficiency Optimization:** The mechanism aims to maximize network efficiency by matching the most suitable miners + to each task, ensuring optimal use of the network's computational power. + +5. **Load Balancing:** It also incorporates load balancing strategies to prevent overburdening individual miners, + thereby maintaining a healthy and sustainable network ecosystem. + +Through these functionalities, the allocation mechanism ensures that computational resources are utilized efficiently +and effectively, contributing to the overall robustness and performance of the network. + +Validators can send requests to reserve access to resources from miners by specifying the specs manually in the +in `register.py` and running this script: https://github.com/neuralinternet/Compute-Subnet/blob/main/neurons/register.py +for example: +```{'cpu':{'count':1}, 'gpu':{'count':1}, 'hard_disk':{'capacity':10737418240}, 'ram':{'capacity':1073741824}}``` + +## Options + +All the list arguments are now using coma separator. + +- `--netuid`: (Optional) The chain subnet uid. Default: 27. +- `--auto_update`: (Optional) Auto update the repository. Default: True. +- `--blacklist.exploiters`: (Optional) Automatically use the list of internal exploiters hotkeys. Default: True. +- `--blacklist.hotkeys `: (Optional) List of hotkeys to blacklist. Default: []. +- `--blacklist.coldkeys `: (Optional) List of coldkeys to blacklist. Default: []. +- `--whitelist.hotkeys `: (Optional) List of hotkeys to whitelist. Default: []. +- `--whitelist.coldkeys `: (Optional) List of coldkeys to whitelist. Default: []. + +## Validators options --- +Flags that you can use with the validator script. + +- `--validator.whitelist.unrecognized`: (Optional) Whitelist the unrecognized miners. Default: False. +- `--validator.perform.hardware.query`: (Optional) Perform the old perfInfo method - useful only as personal benchmark, but it doesn't affect score. Default: False. +- `--validator.challenge.batch.size `: (Optional) Batch size that perform the challenge queries - For lower hardware specifications you might want to use a different batch_size than default. Keep in mind the lower is the batch_size the longer it will take to perform all challenge queries. Default: 64. + +## Miners options + +--- + +- `--miner.hashcat.path `: (Optional) The path of the hashcat binary. Default: hashcat. +- `--miner.hashcat.workload.profile `: (Optional) Performance to apply with hashcat profile: 1 Low, 2 Economic, 3 High, 4 Insane. Run `hashcat -h` for more information. Default: 3. +- `--miner.hashcat.extended.options `: (Optional) Any extra options you found usefull to append to the hascat runner (I'd perhaps recommend -O). Run `hashcat -h` for more information. Default: ''. +- `--miner.whitelist.not.enough.stake`: (Optional) Whitelist the validators without enough stake. Default: False. + +## Benchmarking the machine + +```bash +hashcat -b -m 610 +``` + +Output +``` +Speed.#1.........: 12576.1 MH/s (75.69ms) @ Accel:8 Loops:1024 Thr:1024 Vec:1 +Speed.#2.........: 12576.1 MH/s (75.69ms) @ Accel:8 Loops:1024 Thr:1024 Vec:1 +... +... +``` + +Recommended minimum hashrate for the current difficulty: >= 4500 MH/s. + +Difficulty will increase over time. ## License + This repository is licensed under the MIT License. + ```text # The MIT License (MIT) # Copyright © 2023 Neural Internet diff --git a/compute/__init__.py b/compute/__init__.py index 8036e78..ee219af 100644 --- a/compute/__init__.py +++ b/compute/__init__.py @@ -1,29 +1,73 @@ # The MIT License (MIT) -# Copyright © 2023 - +# Copyright © 2023 Rapiiidooo +# # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the “Software”), to deal in the Software without restriction, including without limitation # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - +# # The above copyright notice and this permission notice shall be included in all copies or substantial portions of # the Software. - +# # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. +import string + # Define the version of the template module. -__version__ = "0.0.0" +__version__ = "1.2.1" +__minimal_validator_version__ = "1.2.0" +__version_signature__ = "393329" + version_split = __version__.split(".") -__spec_version__ = ( - (1000 * int(version_split[0])) - + (10 * int(version_split[1])) - + (1 * int(version_split[2])) -) - -# Import all submodules. -from . import protocol -from . import reward +__version_as_int__ = (100 * int(version_split[0])) + (10 * int(version_split[1])) + (1 * int(version_split[2])) + +# General static vars +validator_permit_stake = 1.024e3 # Stake amount to be a permitted validator + +# Validators static vars +pow_timeout = 60 # Time before the proof of work requests will time out. time unit = seconds +pow_min_difficulty = 5 # Initial and minimal proof of work difficulty. Needs benchmark and adjustment. +pow_max_difficulty = 7 # Maximal proof of work difficulty, this to ensure a miner can not be rewarded for an unlimited unreasonable difficulty. Needs benchmark and adjustment. +pow_default_mode = "610" # Model: BLAKE2b-512($pass.$salt) +pow_default_chars = str(string.ascii_letters + string.digits + "!@#$%^&*()-_+=[]{};:,.<>/") +pow_max_possible_score = 15 # Max reasonable possible score considering the static difficulty, registration and lowest time. + +# Miners static vars +miner_priority_perfinfo = 1 # Lowest priority +miner_priority_challenge = 2 # Medium priority +miner_priority_allocate = 3 # The highest priority +miner_hashcat_location = "hashcat" +miner_hashcat_workload_profile = "3" +miner_whitelist_validator_steps_for = 300 # Number of steps while a validator is whitelisted thanks to his version signature (~5minutes) + +SUSPECTED_EXPLOITERS_COLDKEYS = [] +SUSPECTED_EXPLOITERS_HOTKEYS = [ + "5HZ1ATsziEMDm1iUqNWQatfEDb1JSNf37AiG8s3X4pZzoP3A", + "5H679r89XawDrMhwKGH1jgWMZQ5eeJ8RM9SvUmwCBkNPvSCL", + "5FnMHpqYo1MfgFLax6ZTkzCZNrBJRjoWE5hP35QJEGdZU6ft", + "5H3tiwVEdqy9AkQSLxYaMewwZWDi4PNNGxzKsovRPUuuvALW", + "5E6oa5hS7a6udd9LUUsbBkvzeiWDCgyA2kGdj6cXMFdjB7mm", + "5DFaj2o2R4LMZ2zURhqEeFKXvwbBbAPSPP7EdoErYc94ATP1", + "5H3padRmkFMJqZQA8HRBZUkYY5aKCTQzoR8NwqDfWFdTEtky", + "5HBqT3dhKWyHEAFDENsSCBJ1ntyRdyEDQWhZo1JKgMSrAhUv", + "5FAH7UesJRwwLMkVVknW1rsh9MQMUo78d5Qyx3KpFpL5A7LW", + "5GUJBJmSJtKPbPtUgALn4h34Ydc1tjrNfD1CT4akvcZTz1gE", + "5E2RkNBMCrdfgpnXHuiC22osAxiw6fSgZ1iEVLqWMXSpSKac", + "5DaLy2qQRNsmbutQ7Havj49CoZSKksQSRkCLJsiknH8GcsN2", + "5GNNB5kZfo6F9hqwXvaRfYdTuJPSzrXbtABzwoL499jPNBjt", + "5GVjcJLQboN5NcQoP4x8oqovjAiEizdscoocWo9HBYYmPdR3", + "5FswTe5bbs9n1SzaGpzUd6sDfnzdPfWVS2MwDWNbAneeT15k", + "5F4bqDZkx79hCxmbbsVMuq312EW9hQLvsBzKsAJgcEqpb8L9", +] + +# TODO feat(validators): Select & insert initialized & updated score from db. +# TODO feat(Validators): Implement the dynamic difficulty. +# TODO feat(Validators/Miners): Random hashes used for challenge ? maybe not necessary thanks to Blake algo. +# TODO feat(Miners): Remove docker requirement, to support containerized providers. +# TODO tech-debt: Rename Perfinfo into hardware info or specs info (but as long as it is not used to tests the perfs, IMO name is incorrect. +# TODO tech-debt: Run black everywhere. Use Classes instead of big main and unclassified methods. +# TODO tech-debt: Replace miner file to use a Miner class. diff --git a/compute/axon.py b/compute/axon.py new file mode 100644 index 0000000..4d0a623 --- /dev/null +++ b/compute/axon.py @@ -0,0 +1,383 @@ +# The MIT License (MIT) +# Copyright © 2021 Yuma Rao +# Copyright © 2022 Opentensor Foundation +# Copyright © 2023 Opentensor Technologies Inc +# Copyright © 2023 Rapiiidooo +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the “Software”), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +import copy +import json +import uuid +from typing import Optional + +import bittensor +import bittensor.utils.networking as net +import time +import uvicorn +from bittensor import axon, subtensor +from bittensor.axon import FastAPIThreadedServer, AxonMiddleware +from fastapi import FastAPI, APIRouter +from rich.prompt import Confirm +from starlette.requests import Request + +from compute import __version_as_int__ +from compute.prometheus import prometheus_extrinsic +from compute.utils.version import get_local_version + + +def serve_extrinsic( + subtensor: "bittensor.subtensor", + wallet: "bittensor.wallet", + ip: str, + port: int, + protocol: int, + netuid: int, + placeholder1: int = 0, + placeholder2: int = 0, + wait_for_inclusion: bool = False, + wait_for_finalization=True, + prompt: bool = False, +) -> bool: + r"""Subscribes a bittensor endpoint to the subtensor chain. + Args: + wallet (bittensor.wallet): + bittensor wallet object. + ip (str): + endpoint host port i.e. 192.122.31.4 + port (int): + endpoint port number i.e. 9221 + protocol (int): + int representation of the protocol + netuid (int): + network uid to serve on. + placeholder1 (int): + placeholder for future use. + placeholder2 (int): + placeholder for future use. + wait_for_inclusion (bool): + if set, waits for the extrinsic to enter a block before returning true, + or returns false if the extrinsic fails to enter the block within the timeout. + wait_for_finalization (bool): + if set, waits for the extrinsic to be finalized on the chain before returning true, + or returns false if the extrinsic fails to be finalized within the timeout. + prompt (bool): + If true, the call waits for confirmation from the user before proceeding. + Returns: + success (bool): + flag is true if extrinsic was finalized or uncluded in the block. + If we did not wait for finalization / inclusion, the response is true. + """ + # Decrypt hotkey + wallet.hotkey + version = __version_as_int__ + params: "AxonServeCallParams" = { + "version": version, + "ip": net.ip_to_int(ip), + "port": port, + "ip_type": net.ip_version(ip), + "netuid": netuid, + "hotkey": wallet.hotkey.ss58_address, + "coldkey": wallet.coldkeypub.ss58_address, + "protocol": protocol, + "placeholder1": version, + "placeholder2": version, + } + bittensor.logging.debug("Checking axon ...") + neuron = subtensor.get_neuron_for_pubkey_and_subnet(wallet.hotkey.ss58_address, netuid=netuid) + neuron_up_to_date = not neuron.is_null and params == { + "version": neuron.axon_info.version, + "ip": net.ip_to_int(neuron.axon_info.ip), + "port": neuron.axon_info.port, + "ip_type": neuron.axon_info.ip_type, + "netuid": neuron.netuid, + "hotkey": neuron.hotkey, + "coldkey": neuron.coldkey, + "protocol": neuron.axon_info.protocol, + "placeholder1": neuron.axon_info.placeholder1, + "placeholder2": neuron.axon_info.placeholder2, + } + output = params.copy() + output["coldkey"] = wallet.coldkeypub.ss58_address + output["hotkey"] = wallet.hotkey.ss58_address + if neuron_up_to_date: + bittensor.logging.debug(f"Axon already served on: AxonInfo({wallet.hotkey.ss58_address},{ip}:{port}) ") + return True + + if prompt: + output = params.copy() + output["coldkey"] = wallet.coldkeypub.ss58_address + output["hotkey"] = wallet.hotkey.ss58_address + if not Confirm.ask("Do you want to serve axon:\n [bold white]{}[/bold white]".format(json.dumps(output, indent=4, sort_keys=True))): + return False + + bittensor.logging.debug(f"Serving axon with: AxonInfo({wallet.hotkey.ss58_address},{ip}:{port}) -> {subtensor.network}:{netuid}:{version}") + success, error_message = subtensor._do_serve_axon( + wallet=wallet, + call_params=params, + wait_for_finalization=wait_for_finalization, + wait_for_inclusion=wait_for_inclusion, + ) + + if wait_for_inclusion or wait_for_finalization: + if success == True: + bittensor.logging.debug(f"Axon served with: AxonInfo({wallet.hotkey.ss58_address},{ip}:{port}) on {subtensor.network}:{netuid} ") + return True + else: + bittensor.logging.debug(f"Axon failed to served with error: {error_message} ") + return False + else: + return True + + +class ComputeSubnetSubtensor(subtensor): + def __init__( + self, + network: str = None, + config: "bittensor.config" = None, + _mock: bool = False, + log_verbose: bool = True, + ) -> None: + super().__init__( + network=network, + config=config, + _mock=_mock, + log_verbose=log_verbose, + ) + + ################# + #### Serving #### + ################# + def serve( + self, + wallet: "bittensor.wallet", + ip: str, + port: int, + protocol: int, + netuid: int, + placeholder1: int = __version_as_int__, + placeholder2: int = 0, + wait_for_inclusion: bool = False, + wait_for_finalization=True, + prompt: bool = False, + ) -> bool: + serve_extrinsic( + self, + wallet, + ip, + port, + protocol, + netuid, + placeholder1, + placeholder2, + wait_for_inclusion, + wait_for_finalization, + ) + + def serve_prometheus( + self, + wallet: "bittensor.wallet", + port: int, + netuid: int, + wait_for_inclusion: bool = False, + wait_for_finalization: bool = True, + ) -> bool: + return prometheus_extrinsic( + self, + wallet=wallet, + port=port, + netuid=netuid, + wait_for_inclusion=wait_for_inclusion, + wait_for_finalization=wait_for_finalization, + ) + + +class ComputeSubnetAxon(axon): + def info(self) -> "bittensor.AxonInfo": + """Returns the axon info object associated with this axon.""" + return bittensor.AxonInfo( + version=get_local_version(), + ip=self.external_ip, + ip_type=4, + port=self.external_port, + hotkey=self.wallet.hotkey.ss58_address, + coldkey=self.wallet.coldkeypub.ss58_address, + protocol=4, + placeholder1=1, + placeholder2=2, + ) + + def __init__( + self, + wallet: "bittensor.wallet" = None, + config: Optional["bittensor.config"] = None, + port: Optional[int] = None, + ip: Optional[str] = None, + external_ip: Optional[str] = None, + external_port: Optional[int] = None, + max_workers: Optional[int] = None, + ) -> "bittensor.axon": + r"""Creates a new bittensor.Axon object from passed arguments. + Args: + config (:obj:`Optional[bittensor.config]`, `optional`): + bittensor.axon.config() + wallet (:obj:`Optional[bittensor.wallet]`, `optional`): + bittensor wallet with hotkey and coldkeypub. + port (:type:`Optional[int]`, `optional`): + Binding port. + ip (:type:`Optional[str]`, `optional`): + Binding ip. + external_ip (:type:`Optional[str]`, `optional`): + The external ip of the server to broadcast to the network. + external_port (:type:`Optional[int]`, `optional`): + The external port of the server to broadcast to the network. + max_workers (:type:`Optional[int]`, `optional`): + Used to create the threadpool if not passed, specifies the number of active threads servicing requests. + """ + # Build and check config. + if config is None: + config = axon.config() + config = copy.deepcopy(config) + config.axon.ip = ip or config.axon.get("ip", bittensor.defaults.axon.ip) + config.axon.port = port or config.axon.get("port", bittensor.defaults.axon.port) + config.axon.external_ip = external_ip or config.axon.get("external_ip", bittensor.defaults.axon.external_ip) + config.axon.external_port = external_port or config.axon.get("external_port", bittensor.defaults.axon.external_port) + config.axon.max_workers = max_workers or config.axon.get("max_workers", bittensor.defaults.axon.max_workers) + axon.check_config(config) + self.config = config + + # Get wallet or use default. + self.wallet = wallet or bittensor.wallet() + + # Build axon objects. + self.uuid = str(uuid.uuid1()) + self.ip = self.config.axon.ip + self.port = self.config.axon.port + self.external_ip = self.config.axon.external_ip if self.config.axon.external_ip != None else bittensor.utils.networking.get_external_ip() + self.external_port = self.config.axon.external_port if self.config.axon.external_port != None else self.config.axon.port + self.full_address = str(self.config.axon.ip) + ":" + str(self.config.axon.port) + self.started = False + + # Build middleware + self.thread_pool = bittensor.PriorityThreadPoolExecutor(max_workers=self.config.axon.max_workers) + self.nonces = {} + + # Request default functions. + self.forward_class_types = {} + self.blacklist_fns = {} + self.priority_fns = {} + self.forward_fns = {} + self.verify_fns = {} + self.required_hash_fields = {} + + # Instantiate FastAPI + self.app = FastAPI() + log_level = "trace" if bittensor.logging.__trace_on__ else "critical" + self.fast_config = uvicorn.Config(self.app, host="0.0.0.0", port=self.config.axon.port, log_level=log_level) + self.fast_server = FastAPIThreadedServer(config=self.fast_config) + self.router = APIRouter() + self.app.include_router(self.router) + + # Build ourselves as the middleware. + self.app.add_middleware(ComputeSubnetAxonMiddleware, axon=self) + + # Attach default forward. + def ping(r: bittensor.Synapse) -> bittensor.Synapse: + return r + + self.attach(forward_fn=ping, verify_fn=None, blacklist_fn=None, priority_fn=None) + + +class ComputeSubnetAxonMiddleware(AxonMiddleware): + """ + The `AxonMiddleware` class is a key component in the Axon server, responsible for processing all + incoming requests. It handles the essential tasks of verifying requests, executing blacklist checks, + running priority functions, and managing the logging of messages and errors. Additionally, the class + is responsible for updating the headers of the response and executing the requested functions. + + This middleware acts as an intermediary layer in request handling, ensuring that each request is + processed according to the defined rules and protocols of the Bittensor network. It plays a pivotal + role in maintaining the integrity and security of the network communication. + + Args: + app (FastAPI): An instance of the FastAPI application to which this middleware is attached. + axon (bittensor.axon): The Axon instance that will process the requests. + + The middleware operates by intercepting incoming requests, performing necessary preprocessing + (like verification and priority assessment), executing the request through the Axon's endpoints, and + then handling any postprocessing steps such as response header updating and logging. + """ + + def __init__(self, app: "AxonMiddleware", axon: "bittensor.axon"): + """ + Initialize the AxonMiddleware class. + + Args: + app (object): An instance of the application where the middleware processor is used. + axon (object): The axon instance used to process the requests. + """ + super().__init__(app, axon=axon) + + async def preprocess(self, request: Request) -> bittensor.Synapse: + """ + Performs the initial processing of the incoming request. This method is responsible for + extracting relevant information from the request and setting up the Synapse object, which + represents the state and context of the request within the Axon server. + + Args: + request (Request): The incoming request to be preprocessed. + + Returns: + bittensor.Synapse: The Synapse object representing the preprocessed state of the request. + + The preprocessing involves: + 1. Extracting the request name from the URL path. + 2. Creating a Synapse instance from the request headers using the appropriate class type. + 3. Filling in the Axon and Dendrite information into the Synapse object. + 4. Signing the Synapse from the Axon side using the wallet hotkey. + + This method sets the foundation for the subsequent steps in the request handling process, + ensuring that all necessary information is encapsulated within the Synapse object. + """ + # Extracts the request name from the URL path. + request_name = request.url.path.split("/")[1] + + # Creates a synapse instance from the headers using the appropriate forward class type + # based on the request name obtained from the URL path. + synapse = self.axon.forward_class_types[request_name].from_headers(request.headers) + synapse.name = request_name + + # Fills the local axon information into the synapse. + synapse.axon.__dict__.update( + { + "version": __version_as_int__, + "uuid": str(self.axon.uuid), + "nonce": f"{time.monotonic_ns()}", + "status_message": "Success", + "status_code": "100", + "placeholder1": 1, + "placeholder2": 2, + } + ) + + # Fills the dendrite information into the synapse. + synapse.dendrite.__dict__.update({"port": str(request.client.port), "ip": str(request.client.host)}) + + # Signs the synapse from the axon side using the wallet hotkey. + message = f"{synapse.axon.nonce}.{synapse.dendrite.hotkey}.{synapse.axon.hotkey}.{synapse.axon.uuid}" + synapse.axon.signature = f"0x{self.axon.wallet.hotkey.sign(message).hex()}" + + # Return the setup synapse. + return synapse diff --git a/compute/prometheus.py b/compute/prometheus.py new file mode 100644 index 0000000..bf195a3 --- /dev/null +++ b/compute/prometheus.py @@ -0,0 +1,115 @@ +# The MIT License (MIT) +# Copyright © 2021 Yuma Rao +# Copyright © 2023 Opentensor Foundation +# Copyright © 2023 Rapiiidooo +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the “Software”), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +import bittensor +import bittensor.utils.networking as net + +import compute + + +def prometheus_extrinsic( + subtensor: "bittensor.subtensor", + wallet: "bittensor.wallet", + port: int, + netuid: int, + ip: int = None, + wait_for_inclusion: bool = False, + wait_for_finalization=True, +) -> bool: + r"""Subscribes a bittensor endpoint to the subtensor chain. + Args: + subtensor (bittensor.subtensor): + bittensor subtensor object. + wallet (bittensor.wallet): + bittensor wallet object. + ip (str): + endpoint host port i.e. 192.122.31.4 + port (int): + endpoint port number i.e. 9221 + netuid (int): + network uid to serve on. + wait_for_inclusion (bool): + if set, waits for the extrinsic to enter a block before returning true, + or returns false if the extrinsic fails to enter the block within the timeout. + wait_for_finalization (bool): + if set, waits for the extrinsic to be finalized on the chain before returning true, + or returns false if the extrinsic fails to be finalized within the timeout. + Returns: + success (bool): + flag is true if extrinsic was finalized or included in the block. + If we did not wait for finalization / inclusion, the response is true. + """ + + # ---- Get external ip ---- + if ip == None: + try: + external_ip = net.get_external_ip() + bittensor.logging.trace("Found external ip: {}".format(external_ip)) + except Exception as E: + raise RuntimeError("Unable to attain your external ip. Check your internet connection. error: {}".format(E)) from E + else: + external_ip = ip + + call_params: "PrometheusServeCallParams" = { + "version": compute.__version_as_int__, + "ip": net.ip_to_int(external_ip), + "port": port, + "ip_type": net.ip_version(external_ip), + } + + bittensor.logging.info("Checking Prometheus...") + neuron = subtensor.get_neuron_for_pubkey_and_subnet(wallet.hotkey.ss58_address, netuid=netuid) + + curr_block = subtensor.block - (subtensor.block % 100) + last_update = curr_block - neuron.last_update + if last_update > 100: + bittensor.logging.info("Needs to re-update neuron...") + neuron_up_to_date = None + else: + bittensor.logging.info("Neuron has been updated less than 100 blocks ago...") + neuron_up_to_date = not neuron.is_null and call_params == { + "version": compute.__version_as_int__, + "ip": net.ip_to_int(neuron.prometheus_info.ip), + "port": neuron.prometheus_info.port, + "ip_type": neuron.prometheus_info.ip_type, + } + + if neuron_up_to_date: + bittensor.logging.info(f"Prometheus already Served.") + return True + + # Add netuid, not in prometheus_info + call_params["netuid"] = netuid + + bittensor.logging.info("Serving prometheus on: {}:{} ...".format(subtensor.network, netuid)) + success, err = subtensor._do_serve_prometheus( + wallet=wallet, + call_params=call_params, + wait_for_finalization=wait_for_finalization, + wait_for_inclusion=wait_for_inclusion, + ) + + if wait_for_inclusion or wait_for_finalization: + if success == True: + return True + else: + bittensor.logging.error("Failed to serve prometheus error: {}".format(err)) + return False + else: + return True diff --git a/compute/protocol.py b/compute/protocol.py index b0c1cda..a5b8a88 100644 --- a/compute/protocol.py +++ b/compute/protocol.py @@ -17,7 +17,8 @@ import bittensor as bt -class PerfInfo( bt.Synapse ): + +class PerfInfo(bt.Synapse): """ A simple performance information protocol representation which uses bt.Synapse as its base. This protocol helps in handling performance information request and response communication between @@ -28,9 +29,9 @@ class PerfInfo( bt.Synapse ): - perf_output: A dictionary with the detailed information of cpu, gpu, hard disk and ram. """ - perf_input: str = '' + perf_input: str = "" - perf_output: str = '' + perf_output: str = "" """ Request output, filled by recieving axon. Example: {"CPU":{'count' : 4, 'vendor_id_raw' : 'AuthenticAMD', ...}} @@ -54,7 +55,8 @@ def deserialize(self) -> str: """ return self.perf_output -class Allocate( bt.Synapse ): + +class Allocate(bt.Synapse): """ A simple Allocate protocol representation which uses bt.Synapse as its base. This protocol helps in handling Allocate request and response communication between @@ -84,10 +86,35 @@ def deserialize(self) -> dict: - dict: The deserialized response, which in this case is the value of output. Example: - Assuming a Allocate instance has a output value of {}: + Assuming a Allocate instance has an output value of {}: >>> allocate_instance = Allocate() >>> allocate_instance.output = {} >>> allocate_instance.deserialize() {} """ - return self.output \ No newline at end of file + return self.output + + +class Challenge(bt.Synapse): + # Query parameters + challenge_hash: str = "" + challenge_salt: str = "" + challenge_mode: str = "" + challenge_chars: str = "" + challenge_mask: str = "" + + output: dict = {} + + def deserialize(self) -> dict: + """ + Returns: + - dict: The deserialized response, which in this case is the value of output. + + Example: + Assuming a Challenge instance has an output value of {}: + >>> challenge_instance = Challenge() + >>> challenge_instance.output = {} + >>> challenge_instance.deserialize() + {"password": None, "error": f"Hashcat execution failed with code {process.returncode}: {stderr}"} + """ + return self.output diff --git a/compute/utils/__init__.py b/compute/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/compute/utils/parser.py b/compute/utils/parser.py new file mode 100644 index 0000000..06b6af5 --- /dev/null +++ b/compute/utils/parser.py @@ -0,0 +1,128 @@ +import argparse + +import bittensor as bt + +from compute import miner_hashcat_location, miner_hashcat_workload_profile + + +class ComputeArgPaser(argparse.ArgumentParser): + def __init__(self, description=None): + super().__init__(description=description) + self.add_argument( + "--netuid", + type=int, + default=27, + help="The chain subnet uid.", + ) + self.add_argument( + "--auto_update", + action="store_true", + default=True, + help="Auto update the git repository.", + ) + self.add_argument( + "--blacklist.exploiters", + dest="blacklist_exploiters", + default=True, + action="store_true", + help="Automatically use the list of internal exploiters hotkeys.", + ) + self.add_argument( + "--blacklist.hotkeys", + type=self.parse_list, + dest="blacklist_hotkeys", + help="List of hotkeys to blacklist. Default: [].", + default=[], + ) + self.add_argument( + "--blacklist.coldkeys", + type=self.parse_list, + dest="blacklist_coldkeys", + help="List of coldkeys to blacklist. Default: [].", + default=[], + ) + self.add_argument( + "--whitelist.hotkeys", + type=self.parse_list, + dest="whitelist_hotkeys", + help="List of hotkeys to whitelist. Default: [].", + default=[], + ) + self.add_argument( + "--whitelist.coldkeys", + type=self.parse_list, + dest="whitelist_coldkeys", + help="List of coldkeys to whitelist. Default: [].", + default=[], + ) + + self.add_validator_argument() + self.add_miner_argument() + + # Adds subtensor specific arguments i.e. --subtensor.chain_endpoint ... --subtensor.network ... + bt.subtensor.add_args(self) + # Adds logging specific arguments i.e. --logging.debug ..., --logging.trace .. or --logging.logging_dir ... + bt.logging.add_args(self) + # Adds wallet specific arguments i.e. --wallet.name ..., --wallet.hotkey ./. or --wallet.path ... + bt.wallet.add_args(self) + # Adds axon specific arguments i.e. --axon.port ... + bt.axon.add_args(self) + + self.config = bt.config(self) + + def add_validator_argument(self): + self.add_argument( + "--validator.whitelist.unrecognized", + action="store_true", + dest="whitelist_unrecognized", + help="Whitelist the unrecognized miners. Default: False.", + default=False, + ) + self.add_argument( + "--validator.perform.hardware.query", + action="store_true", + dest="validator_perform_hardware_query", + help="Perform the old perfInfo method - useful only as personal benchmark, but it doesn't affect score.", + default=False, + ) + self.add_argument( + "--validator.challenge.batch.size", + type=int, + dest="validator_challenge_batch_size", + help="For lower hardware specifications you might want to use a different batch_size.", + default=64, + ) + + def add_miner_argument(self): + self.add_argument( + "--miner.hashcat.path", + type=str, + dest="miner_hashcat_path", + help="The path of the hashcat binary.", + default=miner_hashcat_location, + ) + self.add_argument( + "--miner.hashcat.workload.profile", + type=str, + dest="miner_hashcat_workload_profile", + help="Performance to apply with hashcat profile: 1 Low, 2 Economic, 3 High, 4 Insane. Run `hashcat -h` for more information.", + default=miner_hashcat_workload_profile, + ) + self.add_argument( + "--miner.hashcat.extended.options", + type=str, + dest="miner_hashcat_extended_options", + help="Any extra options you found usefull to append to the hascat runner (I'd perhaps recommend -O). Run `hashcat -h` for more information.", + default="", + ) + self.add_argument( + "--miner.whitelist.not.enough.stake", + action="store_true", + dest="miner_whitelist_not_enough_stake", + help="Whitelist the validators without enough stake. Default: False.", + default=False, + ) + + @staticmethod + def parse_list(arg): + return arg.split(",") diff --git a/compute/reward.py b/compute/utils/subtensor.py similarity index 64% rename from compute/reward.py rename to compute/utils/subtensor.py index cd63b40..f934b8c 100644 --- a/compute/reward.py +++ b/compute/utils/subtensor.py @@ -1,33 +1,30 @@ # The MIT License (MIT) -# Copyright © 2023 Yuma Rao -# TODO(developer): Set your name -# Copyright © 2023 - +# Copyright © 2023 Rapiiidooo +# # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the “Software”), to deal in the Software without restriction, including without limitation # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - +# # The above copyright notice and this permission notice shall be included in all copies or substantial portions of # the Software. - +# # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. -import typing -import bittensor as bt - -def dummy(query: int, response: int) -> float: - """ - Reward the miner response to the dummy request. This method returns a reward - value for the miner, which is used to update the miner's score. +import bittensor as bt - Returns: - - float: The reward value for the miner. - """ - return 1.0 if response == query * 2 else 0 +def is_registered(wallet, metagraph, subtensor, entity: str = "validator"): + if wallet.hotkey.ss58_address not in metagraph.hotkeys: + bt.logging.error(f"\nYour {entity}: {wallet} is not registered to chain connection: {subtensor} \nRun btcli register and try again.") + exit() + else: + # Each miner gets a unique identity (UID) in the network for differentiation. + my_subnet_uid = metagraph.hotkeys.index(wallet.hotkey.ss58_address) + bt.logging.info(f"Running {entity} on uid: {my_subnet_uid}") + return my_subnet_uid diff --git a/compute/utils/version.py b/compute/utils/version.py new file mode 100644 index 0000000..9853a0c --- /dev/null +++ b/compute/utils/version.py @@ -0,0 +1,169 @@ +""" +The MIT License (MIT) +Copyright © 2023 demon + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +documentation files (the “Software”), to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, +and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of +the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +""" +import codecs +import os +import re +import subprocess +from os import path + +import bittensor as bt +import git +import requests +import sys + + +def version2number(version: str): + if version and type(version) is str: + version = version.split(".") + return (100 * int(version[0])) + (10 * int(version[1])) + (1 * int(version[2])) + return None + + +def get_remote_version(pattern: str = "__version__"): + url = "https://raw.githubusercontent.com/neuralinternet/Compute-Subnet/main/compute/__init__.py" + response = requests.get(url) + + if response.status_code == 200: + lines = response.text.split("\n") + for line in lines: + if line.startswith(pattern): + version_info = line.split("=")[1].strip(" \"'").replace('"', "") + return version_info + else: + print("Failed to get file content") + return 0 + + +def get_local_version(): + try: + # loading version from __init__.py + here = path.abspath(path.dirname(__file__)) + parent = here.rsplit("/", 1)[0] + with codecs.open(os.path.join(parent, "__init__.py"), encoding="utf-8") as init_file: + version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", init_file.read(), re.M) + version_string = version_match.group(1) + return version_string + except Exception as e: + bt.logging.error(f"Error getting local version. : {e}") + return "" + + +def check_version_updated(): + remote_version = get_remote_version() + local_version = get_local_version() + bt.logging.info(f"Version check - remote_version: {remote_version}, local_version: {local_version}") + + if version2number(remote_version) != version2number(local_version): + bt.logging.info(f"👩‍👦Update to the latest version is required") + return True + else: + return False + + +def update_repo(): + try: + repo = git.Repo(search_parent_directories=True) + + origin = repo.remotes.origin + + if repo.is_dirty(untracked_files=True): + bt.logging.error("Update failed: Uncommited changes detected. Please commit changes or run `git stash`") + return False + try: + bt.logging.info("Try pulling remote repository") + origin.pull() + bt.logging.info("pulling success") + return True + except git.exc.GitCommandError as e: + bt.logging.info(f"update : Merge conflict detected: {e} Recommend you manually commit changes and update") + return handle_merge_conflict(repo) + + except Exception as e: + bt.logging.error(f"update failed: {e} Recommend you manually commit changes and update") + + return False + + +def handle_merge_conflict(repo): + try: + repo.git.reset("--merge") + origin = repo.remotes.origin + current_branch = repo.active_branch + origin.pull(current_branch.name) + + for item in repo.index.diff(None): + file_path = item.a_path + bt.logging.info(f"Resolving conflict in file: {file_path}") + repo.git.checkout("--theirs", file_path) + repo.index.commit("Resolved merge conflicts automatically") + bt.logging.info(f"Merge conflicts resolved, repository updated to remote state.") + bt.logging.info(f"✅ Repo update success") + return True + except git.GitCommandError as e: + bt.logging.error(f"update failed: {e} Recommend you manually commit changes and update") + return False + + +def restart_app(): + bt.logging.info("👩‍🦱app restarted due to the update") + + python = sys.executable + os.execl(python, python, *sys.argv) + + +def try_update_packages(): + bt.logging.info("Try updating packages...") + + try: + repo = git.Repo(search_parent_directories=True) + repo_path = repo.working_tree_dir + + requirements_path = os.path.join(repo_path, "requirements.txt") + + python_executable = sys.executable + subprocess.check_call([python_executable], "-m", "pip", "install", "-r", requirements_path) + subprocess.check_call([python_executable], "-m", "pip", "install", "-e", ".") + bt.logging.info("📦Updating packages finished.") + + except Exception as e: + bt.logging.info(f"Updating packages failed {e}") + + +def try_update(): + try: + if check_version_updated() == True: + bt.logging.info("found the latest version in the repo. try ♻️update...") + if update_repo() == True: + try_update_packages() + restart_app() + except Exception as e: + bt.logging.info(f"Try updating failed {e}") + + +def check_hashcat_version(hashcat_path: str = "hashcat"): + try: + process = subprocess.run([hashcat_path, "--version"], capture_output=True, check=True) + if process and process.stdout: + bt.logging.info(f"Version of hashcat found: {process.stdout.decode()}") + return True + except subprocess.CalledProcessError: + bt.logging.error( + f"Hashcat is not available nor installed on the machine. Please make sure hashcat is available in your PATH or give the explicit location using the following argument: --miner.hashcat.path" + ) + exit() diff --git a/docs/hardware_scoring.md b/docs/hardware_scoring.md new file mode 100644 index 0000000..761e093 --- /dev/null +++ b/docs/hardware_scoring.md @@ -0,0 +1,106 @@ +1. **Score Calculation Function:** + - The `score` function aggregates performance data from different hardware components. + - It calculates individual scores for CPU, GPU, hard disk, and RAM. + - These scores are then weighted and summed to produce a total score. + - A registration bonus is applied if the miner is registered, enhancing the total score. + +2. **Component-Specific Scoring Functions:** + - `get_cpu_score`, `get_gpu_score`, `get_hard_disk_score`, and `get_ram_score` are functions dedicated to + calculating scores for each respective hardware component. + - These functions consider the count, frequency, capacity, and speed of each component, applying a specific level + value for normalization. + - The scores are derived based on the efficiency and capacity of the hardware. + +3. **Weight Assignment:** + - The script defines weights for each hardware component's score, signifying their importance in the overall + performance. + - GPU has the highest weight (0.55), reflecting its significance in mining operations. + - CPU, hard disk, and RAM have lower weights (0.2, 0.1, and 0.15, respectively). + +4. **Registration Check:** + - The `check_if_registered` function verifies if a miner is registered using an external API (`wandb.Api()`). + - Registered miners receive a bonus to their total score, incentivizing official registration in the network. + +5. **Score Aggregation:** + - The individual scores are combined into a numpy array, `score_list`. + - The weights are also arranged in an array, `weight_list`. + - The final score is calculated using a dot product of these arrays, multiplied by 10, and then adjusted with the + registration bonus. + +6. **Handling Multiple CPUs/GPUs:** + - The scoring functions for CPUs (`get_cpu_score`) and GPUs (`get_gpu_score`) are designed to process data that can + represent multiple units. + - For CPUs, the `count` variable in `cpu_info` represents the number of CPU cores or units available. The score is + calculated based on the cumulative capability, taking into account the total count and frequency of all CPU cores. + - For GPUs, similar logic applies. The script can handle data representing multiple GPUs, calculating a cumulative + score based on their collective capacity and speed. + +7. **CPU Scoring (Multiple CPUs):** + - The CPU score is computed by multiplying the total number of CPU cores (`count`) by their average + frequency (`frequency`), normalized against a predefined level. + - This approach ensures that configurations with multiple CPUs are appropriately rewarded for their increased + processing power. + +8. **GPU Scoring (Multiple GPUs):** + - The GPU score is calculated by considering the total capacity (`capacity`) and the average speed (average + of `graphics_speed` and `memory_speed`) of all GPUs in the system. + - The score reflects the aggregate performance capabilities of all GPUs, normalized against a set level. + - This method effectively captures the enhanced computational power provided by multiple GPU setups. + +9. **Aggregated Performance Assessment:** + - The final score calculation in the `score` function integrates the individual scores from CPU, GPU, hard disk, and + RAM. + - This integration allows the scoring system to holistically assess the collective performance of all hardware + components, including scenarios with multiple CPUs and GPUs. + +10. **Implications for Miners:** + +- Miners with multiple GPUs and/or CPUs stand to gain a higher score due to the cumulative calculation of their + hardware's capabilities. +- This approach incentivizes miners to enhance their hardware setup with additional CPUs and GPUs, thereby contributing + more processing power to the network. + +The weight assignments are as follows: + +- **GPU Weight:** 0.55 +- **CPU Weight:** 0.2 +- **Hard Disk Weight:** 0.1 +- **RAM Weight:** 0.15 + +### Example 1: Miner A's Hardware Scores and Weighted Total + +1. **CPU Score:** Calculated as `(2 cores * 3.0 GHz) / 1024 / 50`. +2. **GPU Score:** Calculated as `(8 GB * (1 GHz + 1 GHz) / 2) / 200000`. +3. **Hard Disk Score:** Calculated as `(500 GB * (100 MB/s + 100 MB/s) / 2) / 1000000`. +4. **RAM Score:** Calculated as `(16 GB * 2 GB/s) / 200000`. + +Now, applying the weights: + +- Total Score = (CPU Score × 0.2) + (GPU Score × 0.55) + (Hard Disk Score × 0.1) + (RAM Score × 0.15) +- If registered, add a registration bonus. + +### Example 2: Miner B's Hardware Scores and Weighted Total + +1. **CPU Score:** Calculated as `(4 cores * 2.5 GHz) / 1024 / 50`. +2. **GPU Score:** Calculated as `((6 GB + 6 GB) * (1.5 GHz + 1.2 GHz) / 2) / 200000`. +3. **Hard Disk Score:** Calculated as `(1 TB * (200 MB/s + 150 MB/s) / 2) / 1000000`. +4. **RAM Score:** Calculated as `(32 GB * 3 GB/s) / 200000`. + +Applying the weights: + +- Total Score = (CPU Score × 0.2) + (GPU Score × 0.55) + (Hard Disk Score × 0.1) + (RAM Score × 0.15) +- Since Miner B is not registered, no registration bonus is added. + +### Impact of Weights on Total Score + +- The GPU has the highest weight (0.55), signifying its paramount importance in mining tasks, which are often + GPU-intensive. Miners with powerful GPUs will thus receive a significantly higher portion of their total score from + the GPU component. +- The CPU, while important, has a lower weight (0.2), reflecting its lesser but still vital role in the mining process. +- The hard disk and RAM have the lowest weights (0.1 and 0.15, respectively), indicating their supportive but less + critical roles compared to GPU and CPU. + +It is important to note that the role of validators, in contrast to miners, does not require the integration of GPU +instances. Their function revolves around data integrity and accuracy verification, involving relatively modest network +traffic and lower computational demands. As a result, their hardware requirements are less intensive, focusing more on +stability and reliability rather than high-performance computation. \ No newline at end of file diff --git a/neurons/Miner/container.py b/neurons/Miner/container.py index f5e9705..4b1e79d 100644 --- a/neurons/Miner/container.py +++ b/neurons/Miner/container.py @@ -16,72 +16,107 @@ # DEALINGS IN THE SOFTWARE. # Step 1: Import necessary libraries and modules -import docker import sys import subprocess import json -import os import string import secrets -import bittensor as bt +import rsa import base64 +import platform + +# Function to install Podman based on the available package manager +def install_podman(): + package_managers = { + "apt": ["sudo", "apt", "update", "-y", "&&", "sudo", "apt", "install", "-y", "podman"], + "dnf": ["sudo", "dnf", "install", "-y", "podman"], + "yum": ["sudo", "yum", "install", "-y", "podman"], + # Add more package managers and their commands as needed + } -parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.append(parent_dir) - -import RSAEncryption as rsa - -image_name = "ssh-image" #Docker image name -container_name = "ssh-container" #Docker container name -volume_name = "ssh-volume" #Docker volumne name -volume_path = '/tmp' #Path inside the container where the volume will be mounted -ssh_port = 4444 # Port to map SSH service on the host - -# Initialize Docker client -def get_docker(): - client = docker.from_env() - containers = client.containers.list(all=True) - return client, containers - -# Kill the currently running container -def kill_container(): + system = platform.system() try: - client, containers = get_docker() - running_container = None - for container in containers: - if container_name in container.name: - running_container = container - break - if running_container: - running_container.stop() - running_container.remove() - #bt.logging.info("Container was killed successfully") - return True + if system == "Linux": + distro = platform.linux_distribution(full_distribution_name=False)[0].lower() + for package_manager, command in package_managers.items(): + if subprocess.run(["which", package_manager], stdout=subprocess.PIPE, stderr=subprocess.PIPE).returncode == 0: + print(f"Using {package_manager} to install Podman...") + subprocess.run(command) + return True + print("No compatible package manager found to install Podman.") + return False else: - #bt.logging.info("Unable to find container") + print("Podman installation is only supported on Linux systems.") return False except Exception as e: - #bt.logging.info(f"Error killing container {e}") + print(f"Error installing Podman: {e}") return False - -# Run a new docker container with the given docker_name, image_name and device information -def run_container(cpu_usage, ram_usage, hard_disk_usage, gpu_usage, public_key): +# Check if Podman is installed, and install it if necessary +if not subprocess.run(["which", "podman"], stdout=subprocess.PIPE, stderr=subprocess.PIPE).returncode == 0: + if not install_podman(): + exit("Podman installation failed. Exiting...") + +# Function to generate a random password +def password_generator(length): + characters = string.ascii_letters + string.digits + string.punctuation + return ''.join(secrets.choice(characters) for _ in range(length)) + +# Function to generate a random name +def generate_random_name(prefix): + return f"{prefix}_{secrets.token_hex(4)}" + +# Function to retrieve Podman containers +def get_podman_containers(): + try: + result = subprocess.run(["sudo", "podman", "ps", "-a", "--format", "json"], stdout=subprocess.PIPE, universal_newlines=True, check=True) + containers = json.loads(result.stdout) + return containers + except subprocess.CalledProcessError as e: + print(f"Error retrieving containers: {e}") + return [] + +# Function to stop and remove Podman containers +def stop_and_remove_containers(): + containers = get_podman_containers() + + for container in containers: + container_id = container['ID'] + print(f"Stopping container {container_id}") + stop_podman_container(container_id) + + print(f"Removing container {container_id}") + remove_podman_container(container_id) + +def stop_podman_container(container_id): + try: + subprocess.run(["sudo", "podman", "stop", container_id], check=True) + print(f"Container {container_id} stopped successfully.") + except subprocess.CalledProcessError as e: + print(f"Error stopping container {container_id}: {e}") + +def remove_podman_container(container_id): + try: + subprocess.run(["sudo", "podman", "rm", container_id], check=True) + print(f"Container {container_id} removed successfully.") + except subprocess.CalledProcessError as e: + print(f"Error removing container {container_id}: {e}") + +# Function to run a Podman container +def run_podman_container(): try: - client, containers = get_docker() - # Configuration + # Generate unique names for image and container + image_name = generate_random_name("my_image") + container_name = generate_random_name("my_container") + password = password_generator(10) - cpu_assignment = cpu_usage['assignment'] #e.g : 0-1 - ram_limit = ram_usage['capacity'] # e.g : 5g - hard_disk_capacity = hard_disk_usage['capacity'] # e.g : 100g - gpu_capacity = gpu_usage['capacity'] # e.g : all - # Step 1: Build the Docker image with an SSH server - dockerfile_content = ''' + # Step 1: Create an SSH server in a container using Podman + dockerfile_content = f''' FROM ubuntu RUN apt-get update && apt-get install -y openssh-server RUN mkdir -p /run/sshd # Create the /run/sshd directory - RUN echo 'root:''' + password + '''' | chpasswd + RUN echo 'root:{password}' | chpasswd RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config RUN sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/' /etc/ssh/sshd_config RUN sed -i 's/#ListenAddress 0.0.0.0/ListenAddress 0.0.0.0/' /etc/ssh/sshd_config @@ -92,10 +127,9 @@ def run_container(cpu_usage, ram_usage, hard_disk_usage, gpu_usage, public_key): with open(dockerfile_path, "w") as dockerfile: dockerfile.write(dockerfile_content) - # Build the Docker image - client.images.build(path=os.path.dirname(dockerfile_path), dockerfile=os.path.basename(dockerfile_path), tag=image_name) - # Create the Docker volume with the specified size - #client.volumes.create(volume_name, driver = 'local', driver_opts={'size': hard_disk_capacity}) + # Step 2: Build the image using Podman + build_command = ["sudo", "podman", "build", "-f", dockerfile_path, "-t", image_name, "."] + subprocess.run(build_command, check=True) # Step 2: Run the Docker container container = client.containers.run( @@ -111,55 +145,92 @@ def run_container(cpu_usage, ram_usage, hard_disk_usage, gpu_usage, public_key): ports={22: ssh_port} ) - # Check the status to determine if the container ran successfully - - if container.status == "created": - #bt.logging.info("Container was created successfully") - info = {'username' : 'root', 'password' : password, 'port': ssh_port} - info_str = json.dumps(info) - public_key = public_key.encode('utf-8') - encrypted_info = rsa.encrypt_data(public_key, info_str) - encrypted_info = base64.b64encode(encrypted_info).decode('utf-8') - return {'status' : True, 'info' : encrypted_info} + # Check if the container is created successfully + container_info = subprocess.run(["sudo", "podman", "inspect", container_name], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) + container_data = json.loads(container_info.stdout.decode('utf-8')) + + if container_data and container_data[0]["State"]["Status"] == "running": + info = {'username': 'root', 'password': password, 'port': ssh_port} + return {'status': True, 'info': info, 'image_name': image_name} else: - #bt.logging.info(f"Container falied with status : {container.status}") - return {'status' : False} + return {'status': False} except Exception as e: - #bt.logging.info(f"Error running container {e}") - return {'status' : False} + print(f"Error running container: {e}") + return {'status': False} -# Check if the container exists -def check_container(): +# Function to encrypt data using RSA +def encrypt_rsa(public_key, data): try: - client, containers = get_docker() - for container in containers: - if container_name in container.name: - return True - return False + key = rsa.PublicKey.load_pkcs1(public_key) + encrypted_data = rsa.encrypt(data.encode('utf-8'), key) + return encrypted_data except Exception as e: - #bt.logging.info(f"Error checking container {e}") + print(f"Error encrypting data: {e}") + return None + +# Function to check if a container exists using Podman +def check_container(container_name): + try: + result = subprocess.run(["sudo", "podman", "inspect", container_name], capture_output=True) + return result.returncode == 0 + except Exception as e: + print(f"Error checking container: {e}") return False -# Set the base size of docker, daemon -def set_docker_base_size(base_size):#e.g 100g - docker_daemon_file = "/etc/docker/daemon.json" +# Function to set Podman base size +def set_podman_base_size(base_size): + try: + podman_config_file = "/etc/containers/containers.conf" - # Modify the daemon.json file to set the new base size - storage_options = { - "storage-driver": "devicemapper", - "storage-opts": [ - "dm.basesize=" + base_size - ] - } + # Modify the containers.conf file to set the new base size + storage_options = f"option_storage_driver = 'devicemapper'\noption_storage_opts = ['dm.basesize={base_size}']" - with open(docker_daemon_file, "w") as json_file: - json.dump(storage_options, json_file, indent=4) + with open(podman_config_file, "w") as conf_file: + conf_file.write(storage_options) - # Restart Docker - subprocess.run(["systemctl", "restart", "docker"]) + # Restart Podman service + subprocess.run(["sudo", "systemctl", "restart", "podman"]) + except Exception as e: + print(f"Error setting Podman base size: {e}") -# Randomly generate password for given length +# Function to generate a random password def password_generator(length): - alphabet = string.ascii_letters + string.digits # You can customize this as needed - random_str = ''.join(secrets.choice(alphabet) for _ in range(length)) - return random_str + characters = string.ascii_letters + string.digits + string.punctuation + return ''.join(secrets.choice(characters) for _ in range(length)) + +# Function to generate a random name +def generate_random_name(prefix): + return f"{prefix}_{secrets.token_hex(4)}" + +# Usage example: +# Check if Podman is installed, and install it if necessary +if not subprocess.run(["which", "podman"], stdout=subprocess.PIPE, stderr=subprocess.PIPE).returncode == 0: + if not install_podman(): + exit("Podman installation failed. Exiting...") + +# Get Podman containers and display them +containers = get_podman_containers() +if containers: + print(containers) + +# Stop and remove Podman containers +stop_and_remove_containers() + +# Run a new container +container_info = run_podman_container() +if container_info['status']: + print(f"Image Name: {container_info['image_name']}") + print(f"Container Name: {container_info['container_name']}") + print(f"Password: {container_info['password']}") +else: + print("Failed to run container.") + +# Run a new container with encryption +public_key, private_key = rsa.newkeys(2048) +container_info_encrypted = run_container(container_info['image_name'], container_info['container_name'], "2222", container_info['password']) +if container_info_encrypted['status']: + encrypted_info = rsa.encrypt(json.dumps(container_info_encrypted['info']).encode('utf-8'), public_key) + if encrypted_info: + print(f"Encrypted Info: {base64.b64encode(encrypted_info).decode('utf-8')}") +else: + print("Failed to run container with encryption.") diff --git a/neurons/Miner/kill_container b/neurons/Miner/kill_container deleted file mode 100755 index cadd72d..0000000 Binary files a/neurons/Miner/kill_container and /dev/null differ diff --git a/neurons/Miner/kill_container.py b/neurons/Miner/kill_container.py index 1d5f7dc..845b93e 100644 --- a/neurons/Miner/kill_container.py +++ b/neurons/Miner/kill_container.py @@ -16,27 +16,19 @@ # DEALINGS IN THE SOFTWARE. # Step 1: Import necessary libraries and modules -import docker - -container_name = "ssh-container" #Docker container name - -# Initialize Docker client - -# Kill the currently running container +import subprocess +container_name = "ssh-container" # Podman container name def kill_container(): try: - client = docker.from_env() - containers = client.containers.list(all=True) - running_container = None - for container in containers: - if container_name in container.name: - running_container = container - break - if running_container: - running_container.stop() - running_container.remove() + result_stop = subprocess.run(["podman", "stop", container_name], capture_output=True, text=True) + result_rm = subprocess.run(["podman", "rm", container_name], capture_output=True, text=True) + + if result_stop.returncode == 0 and result_rm.returncode == 0: return True + else: + return False except Exception as e: - #bt.logging.info(f"Error killing container {e}") + print(f"Error killing container: {e}") return False -kill_container() \ No newline at end of file + +kill_container() diff --git a/neurons/Miner/performance.py b/neurons/Miner/performance.py index 508a9b2..929ad8c 100644 --- a/neurons/Miner/performance.py +++ b/neurons/Miner/performance.py @@ -19,15 +19,21 @@ import subprocess import ast import json +import os #Respond the execution of the application def get_respond(app_data): - app_data = ast.literal_eval(app_data) - file_path = './neurons/Miner/app' # Change the file name and extension as needed - - # Write the bytes data to a file - with open(file_path, 'wb') as file: - file.write(app_data) - subprocess.run('chmod +x ' + file_path, shell=True, check=True) - result = subprocess.check_output(file_path, shell=True, text=True) - return result \ No newline at end of file + try: + app_data = ast.literal_eval(app_data) + + main_dir = os.path.dirname(os.path.abspath(__file__)) + file_path = os.path.join(main_dir, 'app') + + # Write the bytes data to a file + with open(file_path, 'wb') as file: + file.write(app_data) + subprocess.run('chmod +x ' + file_path, shell=True, check=True) + result = subprocess.check_output(file_path, shell=True, text=True) + return result + except Exception as e: + return {} diff --git a/neurons/Miner/pow.py b/neurons/Miner/pow.py new file mode 100644 index 0000000..6f64a42 --- /dev/null +++ b/neurons/Miner/pow.py @@ -0,0 +1,138 @@ +# The MIT License (MIT) +# Copyright © 2023 Rapiiidooo +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the “Software”), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +import shlex +import subprocess +from typing import Union + +import bittensor as bt +import time + +import compute + + +def check_cuda_availability(): + import torch + + if torch.cuda.is_available(): + device_count = torch.cuda.device_count() + bt.logging.info(f"CUDA is available with {device_count} CUDA device(s)!") + else: + bt.logging.warning("CUDA is not available or not properly configured on this system.") + + +def hashcat_verify(_hash, output) -> Union[str, None]: + for item in output.split("\n"): + if _hash in item: + return item.strip().split(":")[-1] + return None + + +def run_hashcat( + _hash: str, + salt: str, + mode: str, + chars: str, + mask: str, + timeout: int = compute.pow_timeout, + hashcat_path: str = compute.miner_hashcat_location, + hashcat_workload_profile: str = compute.miner_hashcat_workload_profile, + hashcat_extended_options: str = "", +): + start_time = time.time() + unknown_error_message = f"run_hashcat execution failed" + try: + command = [ + hashcat_path, + f"{_hash}:{salt}", + "-a", + "3", + "-D", + "2", + "-m", + mode, + "-1", + str(chars), + mask, + "-w", + hashcat_workload_profile, + hashcat_extended_options, + ] + command_str = " ".join(shlex.quote(arg) for arg in command) + bt.logging.trace(command_str) + process = subprocess.run(command, capture_output=True, text=True, timeout=timeout) + + execution_time = time.time() - start_time + + # If hashcat returns a valid result + if process.returncode == 0: + if process.stdout: + result = hashcat_verify(_hash, process.stdout) + bt.logging.success(f"Challenge {result} found in {execution_time} seconds !") + return {"password": result, "local_execution_time": execution_time, "error": None} + else: + if process.returncode == 255: + # It means: Already an instance running. + # Retry with new timeout + time.sleep(1) + return run_hashcat( + _hash=_hash, + salt=salt, + mode=mode, + chars=chars, + mask=mask, + timeout=int(timeout - execution_time), + hashcat_path=hashcat_path, + hashcat_workload_profile=hashcat_workload_profile, + hashcat_extended_options=hashcat_extended_options, + ) + error_message = f"Hashcat execution failed with code {process.returncode}: {process.stderr}" + bt.logging.warning(error_message) + return {"password": None, "local_execution_time": execution_time, "error": error_message} + + except subprocess.TimeoutExpired: + execution_time = time.time() - start_time + error_message = f"Hashcat execution timed out" + bt.logging.warning(error_message) + return {"password": None, "local_execution_time": execution_time, "error": error_message} + except Exception as e: + execution_time = time.time() - start_time + bt.logging.warning(f"{unknown_error_message}: {e}") + return {"password": None, "local_execution_time": execution_time, "error": f"{unknown_error_message}: {e}"} + bt.logging.warning(f"{unknown_error_message}: no exceptions") + return {"password": None, "local_execution_time": execution_time, "error": f"{unknown_error_message}: no exceptions"} + + +def run_miner_pow( + _hash: str, + salt: str, + mode: str, + chars: str, + mask: str, + hashcat_path: str = compute.miner_hashcat_location, + hashcat_workload_profile: str = compute.miner_hashcat_workload_profile, + hashcat_extended_options: str = "", +): + result = run_hashcat( + _hash=_hash, + salt=salt, + mode=mode, + chars=chars, + mask=mask, + hashcat_path=hashcat_path, + hashcat_workload_profile=hashcat_workload_profile, + hashcat_extended_options=hashcat_extended_options, + ) + return result diff --git a/neurons/Validator/app_generator.py b/neurons/Validator/app_generator.py index c775209..ac7f311 100644 --- a/neurons/Validator/app_generator.py +++ b/neurons/Validator/app_generator.py @@ -18,28 +18,32 @@ import subprocess import bittensor as bt import re +import os def run(secret_key): - script_name = './neurons/Validator/script.py' - - # Read the content of the script.py file - with open(script_name, 'r') as file: - script_content = file.read() - - # Find and replace the script_key value - - pattern = r"secret_key\s*=\s*.*?#key" - script_content = re.sub(pattern, f"secret_key = {secret_key}#key", script_content, count=1) - - # Write the modified content back to the file - with open(script_name, 'w') as file: - file.write(script_content) - - # Run the pyinstaller command - command = f'cd neurons\ncd Validator\npyinstaller --onefile script.py\ncd ..\ncd ..' try: - subprocess.run(command, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - #bt.logging.info("PyInstaller completed successfully.") - except subprocess.CalledProcessError: - #bt.logging.info("PyInstaller encountered an error.") - return False + main_dir = os.path.dirname(os.path.abspath(__file__)) + script_name = os.path.join(main_dir, 'script.py') + + # Read the content of the script.py file + with open(script_name, 'r') as file: + script_content = file.read() + + # Find and replace the script_key value + + pattern = r"secret_key\s*=\s*.*?#key" + script_content = re.sub(pattern, f"secret_key = {secret_key}#key", script_content, count=1) + + # Write the modified content back to the file + with open(script_name, 'w') as file: + file.write(script_content) + + # Run the pyinstaller command + command = f'cd {main_dir}\npyinstaller --onefile script.py' + try: + subprocess.run(command, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except subprocess.CalledProcessError as e: + bt.logging.error("An error occurred while generating the app.") + bt.logging.error(f"Error output:{e.stderr.decode()}") + except Exception as e: + bt.logging.error(f"{e}") diff --git a/neurons/Validator/calculate_pow_score.py b/neurons/Validator/calculate_pow_score.py new file mode 100644 index 0000000..289c9e0 --- /dev/null +++ b/neurons/Validator/calculate_pow_score.py @@ -0,0 +1,79 @@ +# The MIT License (MIT) +# Copyright © 2023 Crazydevlegend +# Copyright © 2023 Rapiiidooo +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the “Software”), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +# Step 1: Import necessary libraries and modules + +import bittensor as bt +import wandb + +import compute + +__all__ = ["score"] + + +# Calculate score based on the performance information +def score(response, difficulty, hotkey): + try: + success = response["success"] + elapsed_time = response["elapsed_time"] + + if not success: + return 0 + + # Just in case but in theory, it is not possible to fake the difficulty as it is sent by the validator + # Same occurs for the time, it's calculated by the validator so miners can not fake it + difficulty = min(difficulty, compute.pow_max_difficulty) + + # Define base weights for the PoW + difficulty_weight = 0.5 + time_elapsed_weight = 0.5 + + # Apply exponential rewards for difficulty + difficulty_reward = difficulty * (1 + (difficulty**6)) + + # Apply a bonus for registered miners + registration_bonus = check_if_registered(hotkey) * 20000 + + # Modifier for elapsed time effect + time_modifier = 1 / (1 + elapsed_time) * 200000 + + # Calculate the score + final_score = difficulty_reward + (difficulty_weight * difficulty) + (time_elapsed_weight * time_modifier) + registration_bonus + + # Normalize the score + max_score = 1e6 + normalized_score = (final_score / max_score) * 100 + return min(normalized_score, compute.pow_max_possible_score) + except Exception as e: + bt.logging.error(f"An error occurred while calculating score for the following hotkey - {hotkey}: {e}") + return 0 + + +# Check if miner is registered +def check_if_registered(hotkey): + try: + runs = wandb.Api().runs("registered-miners") + values = [] + for run in runs: + if "key" in run.summary: + values.append(run.summary["key"]) + if hotkey in values: + return True + else: + return False + except Exception as _: + return False diff --git a/neurons/Validator/calculate_score.py b/neurons/Validator/calculate_score.py index e97a5cc..b209865 100644 --- a/neurons/Validator/calculate_score.py +++ b/neurons/Validator/calculate_score.py @@ -20,55 +20,71 @@ import bittensor as bt import wandb -#Calculate score based on the performance information +# Calculate score based on the performance information def score(data, hotkey): try: - #Calculate score for each device + # Calculate score for each device cpu_score = get_cpu_score(data["cpu"]) gpu_score = get_gpu_score(data["gpu"]) hard_disk_score = get_hard_disk_score(data["hard_disk"]) ram_score = get_ram_score(data["ram"]) registered = check_if_registered(hotkey) + # Define upper limits for scores + # 128 (max nb cpu) * 5000 (5Ghz) / 1024 (const) / 75 (level) + cpu_limit = 8.33333333333 + # 652472 (capacity) * 16000 (speed Mhz) / 100000 (level) + gpu_limit = 104.39552 + # 10000000000000 (free space 10Tb) * 20000 (speed) / 10000000 (level) + hard_disk_limit = 18.6264514923 + # 512 (free ram 512Gb) * 5000 (speed) / 200000 (level) + ram_limit = 128 + + # Applying upper limits to scores + cpu_score = min(cpu_score, cpu_limit) + gpu_score = min(gpu_score, gpu_limit) + hard_disk_score = min(hard_disk_score, hard_disk_limit) + ram_score = min(ram_score, ram_limit) + score_list = np.array([[cpu_score, gpu_score, hard_disk_score, ram_score]]) - #Define weights for devices - cpu_weight = 0.2 - gpu_weight = 0.55 - hard_disk_weight = 0.1 - ram_weight = 0.15 + # Define weights for devices + cpu_weight = 0.025 + gpu_weight = 0.95 + hard_disk_weight = 0.02 + ram_weight = 0.005 weight_list = np.array([[cpu_weight], [gpu_weight], [hard_disk_weight], [ram_weight]]) - registration_bonus = registered * 10 + registration_bonus = registered * 100 - return np.dot(score_list, weight_list).item() * 10 + registration_bonus + return 10 + np.dot(score_list, weight_list).item() * 100 + registration_bonus except Exception as e: return 0 -#Score of cpu +# Score of cpu def get_cpu_score(cpu_info): try: count = cpu_info['count'] frequency = cpu_info['frequency'] - level = 50 #20, 2.5 + level = 75 # 30, 2.5 return count * frequency / 1024 / level except Exception as e: return 0 -#Score of gpu +# Score of gpu def get_gpu_score(gpu_info): try: - level = 200000 #20GB, 2GHz + level = 100000000 # 10GB, 2GHz capacity = gpu_info['capacity'] / 1024 / 1024 / 1024 speed = (gpu_info['graphics_speed'] + gpu_info['memory_speed']) / 2 return capacity * speed / level except Exception as e: return 0 -#Score of hard disk +# Score of hard disk def get_hard_disk_score(hard_disk_info): try: - level = 1000000 #1TB, 1g/s + level = 10000000 # 1TB, 10g/s capacity = hard_disk_info['free'] / 1024 / 1024 / 1024 speed = (hard_disk_info['read_speed'] + hard_disk_info['write_speed']) / 2 @@ -76,17 +92,17 @@ def get_hard_disk_score(hard_disk_info): except Exception as e: return 0 -#Score of ram +# Score of ram def get_ram_score(ram_info): try: - level = 200000 #100GB, 2g/s - capacity = ram_info['available'] / 1024 / 1024 / 1024 + level = 200000 # 100GB, 2g/s + capacity = ram_info['free'] / 1024 / 1024 / 1024 speed = ram_info['read_speed'] return capacity * speed / level except Exception as e: return 0 -#Check if miner is registered +# Check if miner is registered def check_if_registered(hotkey): try: runs = wandb.Api().runs("registered-miners") @@ -99,5 +115,4 @@ def check_if_registered(hotkey): else: return False except Exception as e: - #bt.logging.info(f"Error getting cpu information : {e}") return False \ No newline at end of file diff --git a/neurons/Validator/database.py b/neurons/Validator/database.py index 47fdf11..839fa8d 100644 --- a/neurons/Validator/database.py +++ b/neurons/Validator/database.py @@ -29,15 +29,15 @@ # Create a table cursor.execute('CREATE TABLE IF NOT EXISTS miner_details (id INTEGER PRIMARY KEY, hotkey TEXT, details TEXT)') -#Fetch hotkeys from database that meets device_requirement +# Fetch hotkeys from database that meets device_requirement def select_miners_hotkey(device_requirement): try: - #Fetch all records from miner_details table + # Fetch all records from miner_details table cursor.execute('CREATE TABLE IF NOT EXISTS tb (id INTEGER PRIMARY KEY, hotkey TEXT, details TEXT)') cursor.execute("SELECT * FROM miner_details") rows = cursor.fetchall() - #Check if the miner meets device_requirement + # Check if the miner meets device_requirement hotkey_list = [] for row in rows: details = json.loads(row[2]) @@ -48,41 +48,49 @@ def select_miners_hotkey(device_requirement): bt.logging.error(f"Error while getting hotkeys from miner_details : {e}") return [] -#Update the miner_details with perfInfo -def update(hotkey_list, perfInfo_responses): +# Update the miner_details with perfInfo +def update(hotkey_list, benchmark_responses): try: cursor.execute(f'DELETE FROM miner_details') - for index, perfInfo in enumerate(perfInfo_responses): - cursor.execute("INSERT INTO miner_details (hotkey, details) VALUES (?, ?)", (hotkey_list[index], json.dumps(perfInfo))) + for index, response in enumerate(benchmark_responses): + cursor.execute("INSERT INTO miner_details (hotkey, details) VALUES (?, ?)", (hotkey_list[index], json.dumps(response))) conn.commit() except Exception as e: bt.logging.error(f"Error while updating miner_details : {e}") -#Check if the miner meets required details +# Check if the miner meets required details def check_if_miner_meet(details, required_details): + if not details: + return False try: - #CPU side + # CPU side cpu_miner = details['cpu'] required_cpu = required_details['cpu'] - if required_cpu and cpu_miner and cpu_miner['count'] <= required_cpu['count']: + if required_cpu and (not cpu_miner or cpu_miner['count'] < required_cpu['count']): return False - - #GPU side + + # GPU side gpu_miner = details['gpu'] required_gpu = required_details['gpu'] - if required_gpu and gpu_miner and gpu_miner['capacity'] <= required_gpu['capacity'] and gpu_miner['count'] <= required_gpu['count']: - return False + if required_gpu: + if not gpu_miner or gpu_miner['capacity'] != required_gpu['capacity'] or gpu_miner['count'] < required_gpu['count']: + return False + else: + gpu_name = str(gpu_miner['details'][0]['name']).lower() + required_type = str(required_gpu['type']).lower() + if required_type not in gpu_name: + return False - #Hard disk side + # Hard disk side hard_disk_miner = details['hard_disk'] required_hard_disk = required_details['hard_disk'] - if required_hard_disk and hard_disk_miner and hard_disk_miner['free'] <= required_hard_disk['capacity']: + if required_hard_disk and (not hard_disk_miner or hard_disk_miner['free'] < required_hard_disk['capacity']): return False - #Ram side + # Ram side ram_miner = details['ram'] required_ram = required_details['ram'] - if required_ram and ram_miner and ram_miner['available'] <= required_ram['capacity']: + if required_ram and (not ram_miner or ram_miner['available'] < required_ram['capacity']): return False except Exception as e: bt.logging.error("The format is wrong, please check it again.") diff --git a/neurons/Validator/pow.py b/neurons/Validator/pow.py new file mode 100644 index 0000000..719eeed --- /dev/null +++ b/neurons/Validator/pow.py @@ -0,0 +1,73 @@ +# The MIT License (MIT) +# Copyright © 2023 Rapiiidooo +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the “Software”), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +import hashlib +import random +import secrets + +import bittensor as bt +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa + +import compute + + +def generate_blake2b_512_hash(data): + hash_object = hashlib.blake2b(data, digest_size=64) + return hash_object.hexdigest() + + +def gen_hash(password): + salt = secrets.token_hex(8) + salted_password = password + salt + data = salted_password.encode("utf-8") + hash_result = hashlib.blake2b(data).hexdigest() + return f"$BLAKE2${hash_result}", salt + + +def gen_random_string(available_chars=compute.pow_default_chars, length=compute.pow_min_difficulty): + # Generating private/public keys + private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048, backend=default_backend()) + private_bytes = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ) + + # Using the private key bytes as seed for guaranteed randomness + seed = int.from_bytes(private_bytes, "big") + random.seed(seed) + return "".join(random.choice(available_chars) for _ in range(length)) + + +def gen_password(length=compute.pow_min_difficulty): + try: + password = gen_random_string(length=length) + _mask = "".join(["?1" for _ in range(length)]) + _hash, _salt = gen_hash(password) + return password, _hash, _salt, _mask + except Exception as e: + bt.logging.error(f"Error during PoW generation (gen_password): {e}") + return None + + +def run_validator_pow(length=compute.pow_min_difficulty): + """ + Don't worry this function is fast enough for validator to use CPUs + """ + password, _hash, _salt, _mask = gen_password(length=length) + return password, _hash, _salt, compute.pow_default_mode, compute.pow_default_chars, _mask diff --git a/neurons/Validator/script.py b/neurons/Validator/script.py index 92cac96..2e973d1 100644 --- a/neurons/Validator/script.py +++ b/neurons/Validator/script.py @@ -16,14 +16,14 @@ # DEALINGS IN THE SOFTWARE. # Step 1: Import necessary libraries and modules import psutil -import igpu +import GPUtil import json import time import subprocess from cryptography.fernet import Fernet -secret_key = b'dCoFT7R_qA3UqKXcGorLm7uZ3dtkUnPe00mvx9pryZc='#key -#Return the detailed information of cpu +secret_key = b'rciNxGPlT5pNNhDIzQ6ABd5nLlFdPMokx_sZIZ8UvwM='#key +# Return the detailed information of cpu def get_cpu_info(): try: # Get the number of physical CPU cores @@ -38,27 +38,25 @@ def get_cpu_info(): return info except Exception as e: - #print(f"Error getting cpu information : {e}") return {} -#Return the detailed information of gpu +# Return the detailed information of gpu def get_gpu_info(): try: - #Count of existing gpus - gpu_count = igpu.count_devices() + # Get existing gpus + gpus = GPUtil.getGPUs() - #Get the detailed information for each gpu (name, capacity) + # Get the detailed information for each gpu (name, capacity) gpu_details = [] capacity = 0 - for i in range(gpu_count): - gpu = igpu.get_device(i) - gpu_details.append({"name" : gpu.name, "capacity" : gpu.memory.total, "utilization" : gpu.utilization.gpu}) - capacity += gpu.memory.total + for gpu in gpus: + gpu_details.append({"name" : gpu.name, "capacity": gpu.memoryTotal}) + capacity += gpu.memoryTotal - info = {"count":gpu_count, "capacity": capacity, "details": gpu_details} + info = {"count":len(gpus), "capacity": capacity, "details": gpu_details} - #Measure speed - if gpu_count: + # Measure speed + if len(gpus): # Run nvidia-smi command to get GPU information result = subprocess.run(['nvidia-smi', '--query-gpu=clocks.gr,clocks.mem', '--format=csv,noheader'], stdout=subprocess.PIPE) gpu_speed_info = result.stdout.decode().strip().split(',') @@ -71,14 +69,13 @@ def get_gpu_info(): return info except Exception as e: - #print(f"Error getting cpu information : {e}") return {} -#Return the detailed information of hard disk +# Return the detailed information of hard disk def get_hard_disk_info(): try: - #Capacity-related information + # Capacity-related information usage = psutil.disk_usage("/") info = {"total": usage.total, "free": usage.free, "used": usage.used} @@ -95,11 +92,8 @@ def get_hard_disk_info(): "free": usage.free }) except Exception as e: - #print(f"Error getting disk information for {partition.device}: {e}") continue - #info["partition"] = partition_info - # Measure write speed size_mb = 100 file_size_bytes = size_mb * 1024 * 1024 @@ -128,10 +122,9 @@ def get_hard_disk_info(): return info except Exception as e: - #print(f"Error getting disk information {e}") return {} -#Return the detailed information of ram +# Return the detailed information of ram def get_ram_info(): try: virtual_memory = psutil.virtual_memory() @@ -147,7 +140,7 @@ def get_ram_info(): "swap_free": swap_memory.free } - #Measure read speed + # Measure read speed size_mb = 100 data = bytearray(size_mb * 1024 * 1024) # Create a byte array of the specified size (in MB) start_time = time.time() @@ -156,7 +149,7 @@ def get_ram_info(): read_time = end_time - start_time read_speed = size_mb / read_time - #Measure write speed + # Measure write speed data = bytearray(size_mb * 1024 * 1024) # Create a byte array of the specified size (in MB) start_time = time.time() data[0:size_mb * 1024 * 1024] = b'\x00' # Write zeros to the entire array in RAM @@ -164,12 +157,11 @@ def get_ram_info(): write_time = end_time - start_time write_speed = size_mb / write_time - info['read_speed'] = read_speed #unit : MB/s - info['write_speed'] = write_speed #unit : MB/s + info['read_speed'] = read_speed # unit : MB/s + info['write_speed'] = write_speed # unit : MB/s return info except Exception as e: - #print(f"Error getting ram information {e}") return {} def get_perf_info(): diff --git a/neurons/miner.py b/neurons/miner.py index 51a3625..7bdc859 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -1,5 +1,6 @@ # The MIT License (MIT) # Copyright © 2023 GitPhantomman +# Copyright © 2023 Rapiiidooo # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the “Software”), to deal in the Software without restriction, including without limitation @@ -15,39 +16,53 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. -# Step 1: Import necessary libraries and modules +import json import os -import sys -import time -import argparse -import typing import traceback +import typing + import bittensor as bt -import Miner.performance as pf +import time + import Miner.allocate as al +import Miner.performance as pf +import Miner.pow as p +import compute +from compute.protocol import PerfInfo, Allocate, Challenge +from compute.utils.parser import ComputeArgPaser +from compute.utils.subtensor import is_registered +from compute.utils.version import get_remote_version, check_hashcat_version, try_update, version2number -parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.append(parent_dir) +whitelist_args_hotkeys_set: set = set() +whitelist_version_hotkeys_set: set = set() +blacklist_args_hotkeys_set: set = set() +exploiters_hotkeys_set: set = set() -import compute def get_config(): - # Step 2: Set up the configuration parser + global whitelist_args_hotkeys_set + global whitelist_version_hotkeys_set + + # Step 1: Set up the configuration parser # This function initializes the necessary command-line arguments. - parser = argparse.ArgumentParser() + parser = ComputeArgPaser(description="This script aims to help miners with the compute subnet.") # Adds override arguments for network and netuid. - parser.add_argument("--netuid", type=int, default=1, help="The chain subnet uid.") - # Adds subtensor specific arguments i.e. --subtensor.chain_endpoint ... --subtensor.network ... - bt.subtensor.add_args(parser) - # Adds logging specific arguments i.e. --logging.debug ..., --logging.trace .. or --logging.logging_dir ... - bt.logging.add_args(parser) - # Adds wallet specific arguments i.e. --wallet.name ..., --wallet.hotkey ./. or --wallet.path ... - bt.wallet.add_args(parser) - # Adds axon specific arguments i.e. --axon.port ... - bt.axon.add_args(parser) + # Activating the parser to read any command-line inputs. config = bt.config(parser) + if config.whitelist_hotkeys: + for hotkey in config.whitelist_hotkeys: + whitelist_args_hotkeys_set.add(hotkey) + + if config.blacklist_hotkeys: + for hotkey in config.blacklist_hotkeys: + blacklist_args_hotkeys_set.add(hotkey) + + if config.blacklist_exploiters: + for key in compute.SUSPECTED_EXPLOITERS_HOTKEYS: + exploiters_hotkeys_set.add(key) + # Step 3: Set up logging directory # Logging captures events for diagnosis or understanding miner's behavior. config.full_path = os.path.expanduser( @@ -65,16 +80,77 @@ def get_config(): return config +def get_valid_queryable_uids(metagraph): + uids = metagraph.uids.tolist() + valid_uids = [] + for index, uid in enumerate(uids): + if metagraph.total_stake[index]: + valid_uids.append(uid) + return valid_uids + + +def get_queryable_axons(metagraph): + queryable_uids = get_valid_queryable_uids(metagraph) + queryable_axons = {metagraph.uids.tolist().index(uid): metagraph.axons[metagraph.uids.tolist().index(uid)] for uid in queryable_uids} + return queryable_axons + + +def get_valid_validator_uids(metagraph: bt.metagraph): + uids = metagraph.uids.tolist() + valid_uids = [] + for index, uid in enumerate(uids): + if metagraph.total_stake[index] > compute.validator_permit_stake: + valid_uids.append(uid) + return valid_uids + + +def get_valid_validator(config, subtensor: bt.subtensor, metagraph: bt.metagraph): + valid_validator_uids = get_valid_validator_uids(metagraph=metagraph) + valid_validator = [] + for uid in valid_validator_uids: + neuron = subtensor.neuron_for_uid(uid, config.netuid) + hotkey = neuron.hotkey + version = neuron.prometheus_info.version + valid_validator.append((uid, hotkey, version)) + + return valid_validator + + +def get_valid_hotkeys(config, subtensor: bt.subtensor, metagraph: bt.metagraph): + whitelist_version_hotkeys_set.clear() + try: + latest_version = version2number(get_remote_version(pattern="__minimal_validator_version__")) + + if latest_version is None: + bt.logging.error(f"Github API call failed or version string is incorrect!") + return + + valid_validators = get_valid_validator(config=config, subtensor=subtensor, metagraph=metagraph) + for uid, hotkey, version in valid_validators: + try: + if version >= latest_version: + bt.logging.debug(f"Version signature match for hotkey : {hotkey}") + whitelist_version_hotkeys_set.add(hotkey) + continue + + bt.logging.debug(f"Version signature mismatch for hotkey : {hotkey}") + except Exception: + bt.logging.error(f"exception in get_valid_hotkeys: {traceback.format_exc()}") + + bt.logging.info(f"Total valid validator hotkeys = {whitelist_version_hotkeys_set}") + + except json.JSONDecodeError: + bt.logging.error(f"exception in get_valid_hotkeys: {traceback.format_exc()}") + + # Main takes the config and starts the miner. def main(config): # Activating Bittensor's logging with the set configurations. bt.logging(config=config, logging_dir=config.full_path) - bt.logging.info( - f"Running miner for subnet: {config.netuid} on network: {config.subtensor.chain_endpoint} with config:" - ) + bt.logging.info(f"Running miner for subnet: {config.netuid} on network: {config.subtensor.chain_endpoint} with config:") # This logs the active configuration to the specified logging directory for review. - #bt.logging.info(config) + # bt.logging.info(config) # Step 4: Initialize Bittensor miner objects # These classes are vital to interact and function within the Bittensor network. @@ -92,11 +168,18 @@ def main(config): metagraph = subtensor.metagraph(config.netuid) bt.logging.info(f"Metagraph: {metagraph}") - if wallet.hotkey.ss58_address not in metagraph.hotkeys: - bt.logging.error( - f"\nYour miner: {wallet} is not registered to chain connection: {subtensor} \nRun btcli register and try again. " - ) - exit() + # Allow validators that are not permitted by stake + miner_whitelist_not_enough_stake = config.miner_whitelist_not_enough_stake + + is_registered(wallet=wallet, metagraph=metagraph, subtensor=subtensor, entity="miner") + + p.check_cuda_availability() + + hashcat_path = config.miner_hashcat_path + hashcat_workload_profile = config.miner_hashcat_workload_profile + hashcat_extended_options = config.miner_hashcat_extended_options + + check_hashcat_version(hashcat_path=hashcat_path) # Each miner gets a unique identity (UID) in the network for differentiation. my_subnet_uid = metagraph.hotkeys.index(wallet.hotkey.ss58_address) @@ -104,67 +187,76 @@ def main(config): # Step 5: Set up miner functionalities # The following functions control the miner's response to incoming requests. + def base_blacklist(synapse: typing.Union[PerfInfo, Allocate, Challenge]) -> typing.Tuple[bool, str]: + hotkey = synapse.dendrite.hotkey + synapse_type = type(synapse).__name__ - # The blacklist function decides if a request should be ignored. - def blacklist_perfInfo(synapse: compute.protocol.PerfInfo) -> typing.Tuple[bool, str]: - if synapse.dendrite.hotkey not in metagraph.hotkeys: + if hotkey not in metagraph.hotkeys: # Ignore requests from unrecognized entities. - bt.logging.trace( - f"Blacklisting unrecognized hotkey {synapse.dendrite.hotkey}" - ) + bt.logging.trace(f"Blacklisting unrecognized hotkey {hotkey}") return True, "Unrecognized hotkey" - bt.logging.trace( - f"Not Blacklisting recognized hotkey {synapse.dendrite.hotkey}" - ) + + index = metagraph.hotkeys.index(hotkey) + stake = metagraph.S[index].item() + + if stake < compute.validator_permit_stake and not miner_whitelist_not_enough_stake: + bt.logging.trace(f"Not enough stake {stake}") + return True, "Not enough stake!" + + if len(whitelist_args_hotkeys_set) > 0 and hotkey not in whitelist_args_hotkeys_set: + return True, "Not whitelisted" + + if len(blacklist_args_hotkeys_set) > 0 and hotkey in blacklist_args_hotkeys_set: + return True, "Blacklisted hotkey" + + # Blacklist entities that are not up-to-date + if hotkey not in whitelist_version_hotkeys_set and len(whitelist_version_hotkeys_set) > 0: + return ( + True, + f"Blacklisted a {synapse_type} request from a non-updated hotkey: {hotkey}", + ) + + if hotkey in exploiters_hotkeys_set: + return ( + True, + f"Blacklisted a {synapse_type} request from an exploiter hotkey: {hotkey}", + ) + + bt.logging.trace(f"Not Blacklisting recognized hotkey {synapse.dendrite.hotkey}") return False, "Hotkey recognized!" + def base_priority(synapse: typing.Union[PerfInfo, Allocate, Challenge]) -> float: + caller_uid = metagraph.hotkeys.index(synapse.dendrite.hotkey) # Get the caller index. + priority = float(metagraph.S[caller_uid]) # Return the stake as the priority. + bt.logging.trace(f"Prioritizing {synapse.dendrite.hotkey} with value: ", priority) + return priority + + # The blacklist function decides if a request should be ignored. + def blacklist_perfInfo(synapse: PerfInfo) -> typing.Tuple[bool, str]: + return base_blacklist(synapse) + # The priority function determines the order in which requests are handled. # More valuable or higher-priority requests are processed before others. - def priority_perfInfo(synapse: compute.protocol.PerfInfo) -> float: - caller_uid = metagraph.hotkeys.index( - synapse.dendrite.hotkey - ) # Get the caller index. - prirority = float(metagraph.S[caller_uid]) # Return the stake as the priority. - bt.logging.trace( - f"Prioritizing {synapse.dendrite.hotkey} with value: ", prirority - ) - return prirority + def priority_perfInfo(synapse: PerfInfo) -> float: + return base_priority(synapse) + compute.miner_priority_perfinfo # This is the PerfInfo function, which decides the miner's response to a valid, high-priority request. - def perfInfo(synapse: compute.protocol.PerfInfo) -> compute.protocol.PerfInfo: - + def perfInfo(synapse: PerfInfo) -> PerfInfo: app_data = synapse.perf_input synapse.perf_output = pf.get_respond(app_data) return synapse - # The blacklist function decides if a request should be ignored. - def blacklist_allocate(synapse: compute.protocol.Allocate) -> typing.Tuple[bool, str]: - if synapse.dendrite.hotkey not in metagraph.hotkeys: - # Ignore requests from unrecognized entities. - bt.logging.trace( - f"Blacklisting unrecognized hotkey {synapse.dendrite.hotkey}" - ) - return True, "Unrecognized hotkey" - bt.logging.trace( - f"Not Blacklisting recognized hotkey {synapse.dendrite.hotkey}" - ) - return False, "Hotkey recognized!" + # The blacklist function decides if a request should be ignored. + def blacklist_allocate(synapse: Allocate) -> typing.Tuple[bool, str]: + return base_blacklist(synapse) # The priority function determines the order in which requests are handled. # More valuable or higher-priority requests are processed before others. - def priority_allocate(synapse: compute.protocol.Allocate) -> float: - caller_uid = metagraph.hotkeys.index( - synapse.dendrite.hotkey - ) # Get the caller index. - prirority = float(metagraph.S[caller_uid]) # Return the stake as the priority. - bt.logging.trace( - f"Prioritizing {synapse.dendrite.hotkey} with value: ", prirority - ) - return prirority + def priority_allocate(synapse: Allocate) -> float: + return base_priority(synapse) + compute.miner_priority_allocate # This is the Allocate function, which decides the miner's response to a valid, high-priority request. - def allocate(synapse: compute.protocol.Allocate) -> compute.protocol.Allocate: - + def allocate(synapse: Allocate) -> Allocate: timeline = synapse.timeline device_requirement = synapse.device_requirement checking = synapse.checking @@ -179,6 +271,31 @@ def allocate(synapse: compute.protocol.Allocate) -> compute.protocol.Allocate: synapse.output = result return synapse + # The blacklist function decides if a request should be ignored. + def blacklist_challenge(synapse: Challenge) -> typing.Tuple[bool, str]: + return base_blacklist(synapse) + + # The priority function determines the order in which requests are handled. + # More valuable or higher-priority requests are processed before others. + def priority_challenge(synapse: Challenge) -> float: + return base_priority(synapse) + compute.miner_priority_challenge + + # This is the Challenge function, which decides the miner's response to a valid, high-priority request. + def challenge(synapse: Challenge) -> Challenge: + bt.logging.info(f"Received challenge (hash, salt): ({synapse.challenge_hash}, {synapse.challenge_salt})") + result = p.run_miner_pow( + _hash=synapse.challenge_hash, + salt=synapse.challenge_salt, + mode=synapse.challenge_mode, + chars=synapse.challenge_chars, + mask=synapse.challenge_mask, + hashcat_path=hashcat_path, + hashcat_workload_profile=hashcat_workload_profile, + hashcat_extended_options=hashcat_extended_options, + ) + synapse.output = result + return synapse + # Step 6: Build and link miner functions to the axon. # The axon handles request processing, allowing validators to send this process requests. axon = bt.axon(wallet=wallet, config=config) @@ -187,20 +304,22 @@ def allocate(synapse: compute.protocol.Allocate) -> compute.protocol.Allocate: # Attach determiners which functions are called when servicing a request. bt.logging.info(f"Attaching forward function to axon.") axon.attach( - forward_fn=perfInfo, - blacklist_fn=blacklist_perfInfo, - priority_fn=priority_perfInfo, - ).attach( forward_fn=allocate, blacklist_fn=blacklist_allocate, priority_fn=priority_allocate, + ).attach( + forward_fn=challenge, + blacklist_fn=blacklist_challenge, + priority_fn=priority_challenge, + ).attach( + forward_fn=perfInfo, + blacklist_fn=blacklist_perfInfo, + priority_fn=priority_perfInfo, ) # Serve passes the axon information to the network + netuid we are hosting on. # This will auto-update if the axon port of external ip have changed. - bt.logging.info( - f"Serving axon {perfInfo, allocate} on network: {config.subtensor.chain_endpoint} with netuid: {config.netuid}" - ) + bt.logging.info(f"Serving axon {perfInfo, allocate, challenge} on network: {config.subtensor.chain_endpoint} with netuid: {config.netuid}") axon.serve(netuid=config.netuid, subtensor=subtensor) # Start starts the miner's axon, making it active on the network. @@ -214,18 +333,27 @@ def allocate(synapse: compute.protocol.Allocate) -> compute.protocol.Allocate: try: # Periodically update our knowledge of the network graph. if step % 5 == 0: - metagraph = subtensor.metagraph(config.netuid) + metagraph.sync(subtensor=subtensor) + log = ( f"Step:{step} | " f"Block:{metagraph.block.item()} | " f"Stake:{metagraph.S[my_subnet_uid]} | " f"Rank:{metagraph.R[my_subnet_uid]} | " - f"Trust:{metagraph.T[my_subnet_uid]} | " + f"Trust:{metagraph.T[my_subnet_uid]} | " f"Consensus:{metagraph.C[my_subnet_uid] } | " f"Incentive:{metagraph.I[my_subnet_uid]} | " f"Emission:{metagraph.E[my_subnet_uid]}" ) bt.logging.info(log) + + if step % compute.miner_whitelist_validator_steps_for == 0: + get_valid_hotkeys(config=config, subtensor=subtensor, metagraph=metagraph) + + # Check for auto update + if step % 100 == 0 and config.auto_update == "yes": + try_update() + step += 1 time.sleep(1) diff --git a/neurons/register.py b/neurons/register.py index 82adc31..602dc86 100644 --- a/neurons/register.py +++ b/neurons/register.py @@ -43,10 +43,10 @@ def get_config(): parser = argparse.ArgumentParser() - # TODO(developer): Adds your custom validator arguments to the parser. - parser.add_argument('--custom', default='my_custom_value', help='Adds a custom value to the parser.') # Adds override arguments for network and netuid. parser.add_argument( '--netuid', type = int, default = 1, help = "The chain subnet uid." ) + parser.add_argument( '--gpu_type', type = str, default = "", help = "The type of GPU required." ) + parser.add_argument( '--gpu_size', type = int, default = 0, help = "The capacity of GPU required in MB." ) # Adds subtensor specific arguments i.e. --subtensor.chain_endpoint ... --subtensor.network ... bt.subtensor.add_args(parser) # Adds logging specific arguments i.e. --logging.debug ..., --logging.trace .. or --logging.logging_dir ... @@ -74,7 +74,7 @@ def get_config(): # Return the parsed config. return config -#Generate ssh connection for given device requirements and timeline +# Generate ssh connection for given device requirements and timeline def allocate (config, device_requirement, timeline, public_key): wallet = bt.wallet( config = config ) bt.logging.info(f"Wallet: {wallet}") @@ -91,7 +91,7 @@ def allocate (config, device_requirement, timeline, public_key): metagraph = subtensor.metagraph( config.netuid ) bt.logging.info(f"Metagraph: {metagraph}") - #Find out the candidates + # Find out the candidates candidates_hotkey = db.select_miners_hotkey(device_requirement) axon_candidates = [] @@ -99,42 +99,29 @@ def allocate (config, device_requirement, timeline, public_key): if axon.hotkey in candidates_hotkey: axon_candidates.append(axon) - allocate_responses = dendrite.query( + responses = dendrite.query( axon_candidates, compute.protocol.Allocate(timeline = timeline, device_requirement = device_requirement, checking = True) ) final_candidates_hotkey = [] - unavailable_uids = [] - for index, allocate_response in enumerate(allocate_responses): + for index, response in enumerate(responses): hotkey = axon_candidates[index].hotkey - if allocate_response and allocate_response['status'] == True: + if response and response['status'] == True: final_candidates_hotkey.append(hotkey) - elif not cs.check_if_registered(hotkey): - uid = metagraph.hotkeys.index(hotkey) - unavailable_uids.append(uid) - - if unavailable_uids: - result = subtensor.set_weights( - netuid = config.netuid, # Subnet to set weights on. - wallet = wallet, # Wallet to sign set weights using hotkey. - uids = unavailable_uids, # Uids of the miners to set weights for. - weights = [0] * len(unavailable_uids), # Weights to set for the miners. - wait_for_inclusion = False - ) - - #Check if there are candidates + + # Check if there are candidates if final_candidates_hotkey == []: return {"status" : False, "msg" : "No proper miner"} - #Sort the candidates with their score + # Sort the candidates with their score scores = torch.ones_like(metagraph.S, dtype=torch.float32) score_dict = {hotkey: score for hotkey, score in zip([axon.hotkey for axon in metagraph.axons], scores)} sorted_hotkeys = sorted(final_candidates_hotkey, key=lambda hotkey: score_dict.get(hotkey, 0), reverse=True) - #Loop the sorted candidates and check if one can allocate the device + # Loop the sorted candidates and check if one can allocate the device for hotkey in sorted_hotkeys: index = metagraph.hotkeys.index(hotkey) axon = metagraph.axons[index] @@ -150,25 +137,10 @@ def allocate (config, device_requirement, timeline, public_key): return {"status" : False, "msg" : "No proper miner"} -#Filter axons with ip address, remove axons with same ip address -def filter_axons_with_ip(axons_list): - # Set to keep track of unique identifiers - unique_ip_addresses = set() - - # List to store filtered axons - filtered_axons = [] - - for axon in axons_list: - ip_address = axon.ip - - if ip_address not in unique_ip_addresses: - unique_ip_addresses.add(ip_address) - filtered_axons.append(axon) - - return filtered_axons - def main( config ): - device_requirement = {'cpu':{'count':1}, 'gpu':{'count': 2, 'capacity': 10737418240}, 'hard_disk':{'capacity':10737418240}, 'ram':{'capacity':10737418240}} + device_requirement = {'cpu':{'count':1}, 'gpu':{}, 'hard_disk':{'capacity':1073741824}, 'ram':{'capacity':1073741824}} + if config.gpu_type != "" and config.gpu_size != 0: + device_requirement['gpu'] = {'count':1, 'capacity':config.gpu_size, 'type':config.gpu_type} timeline = 60 private_key, public_key = rsa.generate_key_pair() result = allocate(config, device_requirement, timeline, public_key) @@ -190,10 +162,10 @@ def upload_wandb(hotkey): except Exception as e: bt.logging.info(f"Error uploading to wandb : {e}") return -# The main function parses the configuration and runs the validator. +# The main function parses the configuration and runs the validator. if __name__ == "__main__": # Parse the configuration. config = get_config() # Run the main function. - main( config ) \ No newline at end of file + main( config ) diff --git a/neurons/validator.py b/neurons/validator.py index f162e52..64e7060 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -1,256 +1,473 @@ # The MIT License (MIT) # Copyright © 2023 Crazydevlegend - +# Copyright © 2023 Rapiiidooo +# # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the “Software”), to deal in the Software without restriction, including without limitation # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, # and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - +# # The above copyright notice and this permission notice shall be included in all copies or substantial portions of # the Software. - +# # THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO # THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. -# Step 1: Import necessary libraries and modules +import ast +import asyncio import os -import sys -import time -import torch -import argparse +import random import traceback -import json +from typing import List, Dict, Union + import bittensor as bt +import time +import torch +from cryptography.fernet import Fernet +from torch._C._te import Tensor + import Validator.app_generator as ag -import Validator.calculate_score as cs +import Validator.calculate_pow_score as cps import Validator.database as db -from cryptography.fernet import Fernet -import ast -import RSAEncryption as rsa -import base64 - -parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.append(parent_dir) - -import compute - -# Step 2: Set up the configuration parser -# This function is responsible for setting up and parsing command-line arguments. -def get_config(): - - parser = argparse.ArgumentParser() - # TODO(developer): Adds your custom validator arguments to the parser. - parser.add_argument('--custom', default='my_custom_value', help='Adds a custom value to the parser.') - # Adds override arguments for network and netuid. - parser.add_argument( '--netuid', type = int, default = 1, help = "The chain subnet uid." ) - # Adds subtensor specific arguments i.e. --subtensor.chain_endpoint ... --subtensor.network ... - bt.subtensor.add_args(parser) - # Adds logging specific arguments i.e. --logging.debug ..., --logging.trace .. or --logging.logging_dir ... - bt.logging.add_args(parser) - # Adds wallet specific arguments i.e. --wallet.name ..., --wallet.hotkey ./. or --wallet.path ... - bt.wallet.add_args(parser) - # Parse the config (will take command-line arguments if provided) - # To print help message, run python3 template/miner.py --help - config = bt.config(parser) - - # Step 3: Set up logging directory - # Logging is crucial for monitoring and debugging purposes. - config.full_path = os.path.expanduser( - "{}/{}/{}/netuid{}/{}".format( - config.logging.logging_dir, - config.wallet.name, - config.wallet.hotkey, - config.netuid, - 'validator', +from Validator.pow import run_validator_pow +from compute import pow_min_difficulty, pow_timeout, SUSPECTED_EXPLOITERS_HOTKEYS, SUSPECTED_EXPLOITERS_COLDKEYS +from compute.axon import ComputeSubnetSubtensor +from compute.protocol import Challenge, PerfInfo, Allocate +from compute.utils.parser import ComputeArgPaser +from compute.utils.subtensor import is_registered +from compute.utils.version import try_update, get_local_version + + +class Validator: + pow_responses: dict = {} + pow_benchmark: dict = {} + + scores: Tensor + + score_decay_factor = 0.334 + score_limit = 0.5 + + _queryable_uids: Dict[int, bt.AxonInfo] + + @property + def wallet(self): + return self._wallet + + @property + def subtensor(self): + return self._subtensor + + @property + def dendrite(self): + return self._dendrite + + @property + def metagraph(self): + return self._metagraph + + @property + def queryable_uids(self): + return [uid for uid in self._queryable_uids.keys()] + + @property + def queryable_axons(self): + return [axon for axon in self._queryable_uids.values()] + + @property + def queryable_hotkeys(self): + return [axon.hotkey for axon in self._queryable_uids.values()] + + def __init__(self): + # Step 1: Parse the bittensor and compute subnet config + self.config = self.init_config() + + # Set up logging with the provided configuration and directory. + bt.logging(config=self.config, logging_dir=self.config.full_path) + bt.logging.info(f"Running validator for subnet: {self.config.netuid} on network: {self.config.subtensor.chain_endpoint} with config:") + # Log the configuration for reference. + bt.logging.info(self.config) + + # Step 2: Build Bittensor validator objects + # These are core Bittensor classes to interact with the network. + bt.logging.info("Setting up bittensor objects.") + + # The wallet holds the cryptographic key pairs for the validator. + self._wallet = bt.wallet(config=self.config) + bt.logging.info(f"Wallet: {self.wallet}") + + # The subtensor is our connection to the Bittensor blockchain. + self._subtensor = ComputeSubnetSubtensor(config=self.config) + bt.logging.info(f"Subtensor: {self.subtensor}") + + # Dendrite is the RPC client; it lets us send messages to other nodes (axons) in the network. + self._dendrite = bt.dendrite(wallet=self.wallet) + bt.logging.info(f"Dendrite: {self.dendrite}") + + # The metagraph holds the state of the network, letting us know about other miners. + self._metagraph = self.subtensor.metagraph(self.config.netuid) + bt.logging.info(f"Metagraph: {self.metagraph}") + + # Set blacklist and whitelist arrays + self.blacklist_hotkeys = {hotkey for hotkey in self.config.blacklist_hotkeys} + self.blacklist_coldkeys = {coldkey for coldkey in self.config.blacklist_coldkeys} + self.whitelist_hotkeys = {hotkey for hotkey in self.config.whitelist_hotkeys} + self.whitelist_coldkeys = {coldkey for coldkey in self.config.whitelist_coldkeys} + + self.exploiters_hotkeys = {hotkey for hotkey in SUSPECTED_EXPLOITERS_HOTKEYS} if self.config.blacklist_exploiters else {} + self.exploiters_coldkeys = {coldkey for coldkey in SUSPECTED_EXPLOITERS_COLDKEYS} if self.config.blacklist_exploiters else {} + + # Set custom validator arguments + self.validator_challenge_batch_size = self.config.validator_challenge_batch_size + self.validator_perform_hardware_query = self.config.validator_perform_hardware_query + + # Step 3: Connect the validator to the network + # Check if hotkey is registered + is_registered(wallet=self.wallet, metagraph=self.metagraph, subtensor=self.subtensor, entity="validator") + + # Initialize the prometheus transaction + self.init_prometheus() + + # Step 4: Set up initial scoring weights for validation + bt.logging.info("Building validation weights.") + self.uids: list = self.metagraph.uids.tolist() + self.last_uids: list = self.uids.copy() + self.init_scores() + + self.curr_block = self.subtensor.block + self.last_updated_block = self.curr_block - (self.curr_block % 100) + + # Init the event loop. + self.loop = asyncio.get_event_loop() + + @staticmethod + def init_config(): + """ + This function is responsible for setting up and parsing command-line arguments. + :return: config + """ + parser = ComputeArgPaser(description="This script aims to help validators with the compute subnet.") + config = parser.config + + # Step 3: Set up logging directory + # Logging is crucial for monitoring and debugging purposes. + config.full_path = os.path.expanduser( + "{}/{}/{}/netuid{}/{}".format( + config.logging.logging_dir, + config.wallet.name, + config.wallet.hotkey, + config.netuid, + "validator", + ) ) - ) - # Ensure the logging directory exists. - if not os.path.exists(config.full_path): os.makedirs(config.full_path, exist_ok=True) - - # Return the parsed config. - return config - -#Filter axons with ip address, remove axons with same ip address -def filter_axons_with_ip(axons_list): - # Set to keep track of unique identifiers - unique_ip_addresses = set() - - # List to store filtered axons - filtered_axons = [] - - for axon in axons_list: - ip_address = axon.ip - - if ip_address not in unique_ip_addresses: - unique_ip_addresses.add(ip_address) - filtered_axons.append(axon) - - return filtered_axons - -def main( config ): - # Set up logging with the provided configuration and directory. - bt.logging(config=config, logging_dir=config.full_path) - bt.logging.info(f"Running validator for subnet: {config.netuid} on network: {config.subtensor.chain_endpoint} with config:") - # Log the configuration for reference. - #bt.logging.info(config) - - # Step 4: Build Bittensor validator objects - # These are core Bittensor classes to interact with the network. - bt.logging.info("Setting up bittensor objects.") - - # The wallet holds the cryptographic key pairs for the validator. - wallet = bt.wallet( config = config ) - bt.logging.info(f"Wallet: {wallet}") - - # The subtensor is our connection to the Bittensor blockchain. - subtensor = bt.subtensor( config = config ) - bt.logging.info(f"Subtensor: {subtensor}") - - # Dendrite is the RPC client; it lets us send messages to other nodes (axons) in the network. - dendrite = bt.dendrite( wallet = wallet ) - bt.logging.info(f"Dendrite: {dendrite}") - - # The metagraph holds the state of the network, letting us know about other miners. - metagraph = subtensor.metagraph( config.netuid ) - bt.logging.info(f"Metagraph: {metagraph}") - - # Step 5: Connect the validator to the network - if wallet.hotkey.ss58_address not in metagraph.hotkeys: - bt.logging.error(f"\nYour validator: {wallet} if not registered to chain connection: {subtensor} \nRun btcli register and try again.") - exit() - else: - # Each miner gets a unique identity (UID) in the network for differentiation. - my_subnet_uid = metagraph.hotkeys.index(wallet.hotkey.ss58_address) - bt.logging.info(f"Running validator on uid: {my_subnet_uid}") - - # Step 6: Set up initial scoring weights for validation - bt.logging.info("Building validation weights.") - alpha = 0.9 - scores = torch.ones_like(metagraph.S, dtype=torch.float32) - - curr_block = subtensor.block - last_updated_block = curr_block - (curr_block % 100) - last_reset_weights_block = curr_block - - # Step 7: The Main Validation Loop - bt.logging.info("Starting validator loop.") - step = 0 - while True: - try: - # TODO(developer): Define how the validator selects a miner to query, how often, etc. - if step % 10 == 1: - - #Filter axons, remove duplicated ip address - axons_list = metagraph.axons - axons_list = filter_axons_with_ip(axons_list) - - #Generate secret key for app - secret_key = Fernet.generate_key() - cipher_suite = Fernet(secret_key) - #Compile the script and generate an exe - ag.run(secret_key) - - #Read the exe file and save it to app_data - with open('neurons//Validator//dist//script', 'rb') as file: - # Read the entire content of the EXE file - app_data = file.read() - - #The responses of PerfInfo request - ret_perfInfo_responses = dendrite.query( - axons_list, - compute.protocol.PerfInfo(perf_input = repr(app_data)), - timeout = 30 - ) - - #Filter invalid responses - perfInfo_responses = [] - hotkey_list = [] - for index, perfInfo in enumerate(ret_perfInfo_responses): - if perfInfo: - binary_data = ast.literal_eval(perfInfo) #Convert str to binary data - decoded_data = ast.literal_eval(cipher_suite.decrypt(binary_data).decode()) #Decrypt data and convert it to object - perfInfo_responses.append(decoded_data) - hotkey_list.append(axons_list[index].hotkey) - - bt.logging.info(f"PerfInfo : {perfInfo_responses}") - - db.update(hotkey_list, perfInfo_responses) - - #Make score_list based on the perf_info - score_list = {} - - for index, perfInfo in enumerate(perfInfo_responses): - hotkey = hotkey_list[index] #hotkey of miner - score_list[hotkey] = cs.score(perfInfo, hotkey) - - #Fill the score_list with 0 for no response miners - for hotkey in metagraph.hotkeys: - if hotkey in score_list: + # Ensure the logging directory exists. + if not os.path.exists(config.full_path): + os.makedirs(config.full_path, exist_ok=True) + + # Return the parsed config. + return config + + def init_prometheus(self): + """ + Register the prometheus information on metagraph. + :return: bool + """ + bt.logging.info("Extrinsic prometheus information on metagraph.") + success = self.subtensor.serve_prometheus( + wallet=self.wallet, + port=bt.defaults.axon.port, + netuid=self.config.netuid, + ) + if success: + bt.logging.success(prefix="Prometheus served", sufix=f"Current version: {get_local_version()}") + else: + bt.logging.error("Prometheus initialization failed") + return success + + def init_scores(self): + self.scores = torch.zeros(len(self.uids), dtype=torch.float32) + bt.logging.info(f"🔢 Initialized scores : {self.scores.tolist()}") + + def sync_scores(self): + # Set the weights of validators to zero. + self.scores = self.scores * (self.metagraph.total_stake < 1.024e3) + # Set the weight to zero for all nodes without assigned IP addresses. + self.scores = self.scores * torch.Tensor(self.get_valid_tensors(metagraph=self.metagraph)) + bt.logging.info(f"🔢 Synced scores : {self.scores.tolist()}") + + def sync_local(self): + """ + Resync our local state with the latest state from the blockchain. + Sync scores with metagraph. + Get the current uids of all miners in the network. + """ + bt.logging.info(f"🔄 Syncing metagraph with subtensor.") + self.sync_scores() + self._metagraph = self.subtensor.metagraph(self.config.netuid) + self.uids = self.metagraph.uids.tolist() + + def start(self): + """The Main Validation Loop""" + # Step 5: Perform queries to miners, scoring, and weight + bt.logging.info("Starting validator loop.") + + step = 0 + step_pseudo_rdm = 20 + while True: + current_block = self.subtensor.block + try: + # Sync the subtensor state with the blockchain, every ~ 1 minute + if step % 10 == 0: + self.sync_local() + + # Perform pow queries, between ~ 10 and 14 minutes + if step % step_pseudo_rdm == 0: + # Prepare the next random step the validators will challenge again + step_pseudo_rdm = random.randint(100, 140) + + # Filter axons with stake and ip address. + self._queryable_uids = self.get_queryable() + + pow_request = {} + + async def run_pow(): + for i in range(0, len(self.uids), self.validator_challenge_batch_size): + tasks = [] + for _uid in self.uids[i : i + self.validator_challenge_batch_size]: + try: + axon = self._queryable_uids[_uid] + password, _hash, _salt, mode, chars, mask = run_validator_pow() + pow_request[_uid] = (password, _hash, _salt, mode, chars, mask, pow_min_difficulty) + tasks.append(self.execute_pow_request(_uid, axon, password, _hash, _salt, mode, chars, mask)) + except KeyError: + continue + await asyncio.gather(*tasks) + + self.loop.run_until_complete(run_pow()) + + # Logs benchmarks for the validators + bt.logging.info("🔢 Results benchmarking:") + for uid, benchmark in self.pow_benchmark.items(): + bt.logging.info(f"{uid}: {benchmark}") + + # TODO update db accordingly with pow results + # db.update(...) + + # Calculate score + score_uid_dict = {} + for uid in self.uids: + previous_score = self.scores[uid] + try: + score = cps.score(self.pow_benchmark[uid], pow_request[uid][-1], self._queryable_uids[uid].hotkey) + except (ValueError, KeyError): + score = 0 + + if previous_score > score < self.score_limit: + decayed_score = previous_score * self.score_decay_factor + else: + decayed_score = score + + self.scores[uid] = decayed_score if decayed_score > self.score_limit else score + score_uid_dict[uid] = self.scores[uid].item() + + bt.logging.info(f"🔢 Updated scores : {score_uid_dict}") + + # ~ every 5 minutes + if step % 50 == 0: + # Check for auto update + if self.config.auto_update: + try_update() + # Frequently check if the validator is still registered + is_registered(wallet=self.wallet, metagraph=self.metagraph, subtensor=self.subtensor, entity="validator") + + # ~ every 20 minutes + if step % 200 == 0 and self.validator_perform_hardware_query: + # # Prepare app_data for benchmarking + # # Generate secret key for app + secret_key = Fernet.generate_key() + cipher_suite = Fernet(secret_key) + # # Compile the script and generate an exe. + ag.run(secret_key) + try: + main_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = os.path.join(main_dir, "Validator/dist/script") + # Read the exe file and save it to app_data. + with open(file_name, "rb") as file: + # Read the entire content of the EXE file + app_data = file.read() + except Exception as e: + bt.logging.error(f"{e}") continue - score_list[hotkey] = 0 - - #Find the maximum score - max_score = score_list[max(score_list, key = score_list.get)] - if max_score == 0: - max_score = 1 - - original_scores = torch.ones_like(metagraph.S, dtype=torch.float32) - - # Calculate score - for index, uid in enumerate(metagraph.uids): - score = score_list[metagraph.neurons[uid].axon_info.hotkey] - # Update the global score of the miner. - # This score contributes to the miner's weight in the network. - # A higher weight means that the miner has been consistently responding correctly. - if len(scores)> index: - scores[index] = alpha * scores[index] + (1 - alpha) * score / max_score - else: - scores[index] = original_scores[index] - - # Periodically update the weights on the Bittensor blockchain. - current_block = subtensor.block - bt.logging.info(f"{current_block}, {last_updated_block}") - if current_block - last_updated_block > 100: - # TODO(developer): Define how the validator normalizes scores before setting weights. - weights = torch.nn.functional.normalize(scores, p=1.0, dim=0) - for i, weight_i in enumerate(weights): - bt.logging.info(f"Weight of Miner{i + 1} : {weight_i}") - # This is a crucial step that updates the incentive mechanism on the Bittensor blockchain. - # Miners with higher scores (or weights) receive a larger share of TAO rewards on this subnet. - result = subtensor.set_weights( - netuid = config.netuid, # Subnet to set weights on. - wallet = wallet, # Wallet to sign set weights using hotkey. - uids = metagraph.uids, # Uids of the miners to set weights for. - weights = weights, # Weights to set for the miners. - wait_for_inclusion = False - ) - last_updated_block = current_block - if result: bt.logging.success('Successfully set weights.') - else: bt.logging.error('Failed to set weights.') - - # End the current step and prepare for the next iteration. - step += 1 - # Resync our local state with the latest state from the blockchain. - metagraph = subtensor.metagraph(config.netuid) - # Sleep for a duration equivalent to the block time (i.e., time between successive blocks). - time.sleep(bt.__blocktime__) - - # If we encounter an unexpected error, log it for debugging. - except RuntimeError as e: - bt.logging.error(e) - traceback.print_exc() - - # If the user interrupts the program, gracefully exit. - except KeyboardInterrupt: - bt.logging.success("Keyboard interrupt detected. Exiting validator.") - exit() - -# The main function parses the configuration and runs the validator. + # Query the miners for benchmarking + bt.logging.info(f"🆔 Hardware list of uids : {self.queryable_uids}") + responses: List[Union[PerfInfo, Allocate, Challenge]] = self.dendrite.query( + self.queryable_axons, + PerfInfo(perf_input=repr(app_data)), + timeout=120, + ) + + # Format responses and save them to benchmark_responses + hardware_list_responses = [] + for index, response in enumerate(responses): + try: + if response: + binary_data = ast.literal_eval(response) # Convert str to binary data + decoded_data = ast.literal_eval(cipher_suite.decrypt(binary_data).decode()) # Decrypt data and convert it to object + hardware_list_responses.append(decoded_data) + else: + hardware_list_responses.append({}) + except Exception as _: + hardware_list_responses.append({}) + + db.update(self.queryable_hotkeys, hardware_list_responses) + bt.logging.info(f"🔢 Hardware list responses : {hardware_list_responses}") + + # Periodically update the weights on the Bittensor blockchain, ~ every 20 minutes + if current_block - self.last_updated_block > 100: + self.set_weights() + self.last_updated_block = current_block + + bt.logging.info(f"Validator running at block {current_block}...") + step += 1 + + # Sleep for a duration equivalent to half a block time (i.e., time between successive blocks). + time.sleep(bt.__blocktime__ / 2) + + # If we encounter an unexpected error, log it for debugging. + except RuntimeError as e: + bt.logging.error(e) + traceback.print_exc() + + # If the user interrupts the program, gracefully exit. + except KeyboardInterrupt: + bt.logging.success("Keyboard interrupt detected. Exiting validator.") + exit() + + def set_weights(self): + weights: torch.FloatTensor = torch.nn.functional.normalize(self.scores, p=1.0, dim=0).float() + bt.logging.info(f"🏋️ Weight of miners : {weights.tolist()}") + # This is a crucial step that updates the incentive mechanism on the Bittensor blockchain. + # Miners with higher scores (or weights) receive a larger share of TAO rewards on this subnet. + result = self.subtensor.set_weights( + netuid=self.config.netuid, # Subnet to set weights on. + wallet=self.wallet, # Wallet to sign set weights using hotkey. + uids=self.metagraph.uids, # Uids of the miners to set weights for. + weights=weights, # Weights to set for the miners. + wait_for_inclusion=False, + ) + if result: + bt.logging.success("Successfully set weights.") + else: + bt.logging.error("Failed to set weights.") + + # Filter the axons with uids_list, remove those with the same IP address. + @staticmethod + def filter_axons(queryable_tuple_uids_axons): + # Set to keep track of unique identifiers + valid_ip_addresses = set() + + # List to store filtered axons + dict_filtered_axons = {} + for uid, axon in queryable_tuple_uids_axons: + ip_address = axon.ip + + if ip_address not in valid_ip_addresses: + valid_ip_addresses.add(ip_address) + dict_filtered_axons[uid] = axon + + return dict_filtered_axons + + def is_blacklisted(self, neuron: bt.NeuronInfoLite): + coldkey = neuron.coldkey + hotkey = neuron.hotkey + + # Blacklist coldkeys that are blacklisted by user + if coldkey in self.blacklist_coldkeys: + bt.logging.trace(f"Blacklisted recognized coldkey {coldkey} - with hotkey: {hotkey}") + return True + + # Blacklist coldkeys that are blacklisted by user or by set of hotkeys + if hotkey in self.blacklist_hotkeys: + bt.logging.trace(f"Blacklisted recognized hotkey {hotkey}") + # Add the coldkey attached to this hotkey in the blacklisted coldkeys + self.blacklist_hotkeys.add(coldkey) + return True + + # Blacklist coldkeys that are exploiters + if coldkey in self.exploiters_coldkeys: + bt.logging.trace(f"Blacklisted exploiter coldkey {coldkey} - with hotkey: {hotkey}") + return True + + # Blacklist hotkeys that are exploiters + if hotkey in self.exploiters_hotkeys: + bt.logging.trace(f"Blacklisted exploiter hotkey {hotkey}") + # Add the coldkey attached to this hotkey in the blacklisted coldkeys + self.exploiters_hotkeys.add(coldkey) + return True + + return False + + def get_valid_queryable(self): + valid_queryable = [] + for uid in self.uids: + neuron: bt.NeuronInfoLite = self.metagraph.neurons[uid] + axon = self.metagraph.axons[uid] + + if neuron.axon_info.ip != "0.0.0.0" and self.metagraph.total_stake[uid] < 1.024e3 and not self.is_blacklisted(neuron=neuron): + valid_queryable.append((uid, axon)) + + return valid_queryable + + def get_valid_tensors(self, metagraph): + tensors = [] + for uid in metagraph.uids: + neuron = metagraph.neurons[uid] + + if neuron.axon_info.ip != "0.0.0.0" and not self.is_blacklisted(neuron=neuron): + tensors.append(True) + else: + tensors.append(False) + + return tensors + + def get_queryable(self): + queryable = self.get_valid_queryable() + dict_filtered_axons = self.filter_axons(queryable_tuple_uids_axons=queryable) + return dict_filtered_axons + + async def execute_pow_request(self, uid, axon, password, _hash, _salt, mode, chars, mask): + start_time = time.time() + response = self.dendrite.query( + axon, + Challenge( + challenge_hash=_hash, + challenge_salt=_salt, + challenge_mode=mode, + challenge_chars=chars, + challenge_mask=mask, + ), + timeout=pow_timeout, + ) + elapsed_time = time.time() - start_time + self.pow_responses[uid] = response + + if password != response.get("password"): + self.pow_benchmark[uid] = {"success": False, "elapsed_time": elapsed_time} + else: + self.pow_benchmark[uid] = {"success": True, "elapsed_time": elapsed_time} + + +def main(): + """ + Main function to run the neuron. + + This function initializes and runs the neuron. It handles the main loop, state management, and interaction + with the Bittensor network. + """ + Validator().start() + + if __name__ == "__main__": - # Parse the configuration. - config = get_config() - # Run the main function. - main( config ) + main() diff --git a/requirements.txt b/requirements.txt index aff110f..075f192 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,3 @@ psutil igpu cryptography pyinstaller -docker \ No newline at end of file