Skip to content

Commit

Permalink
Merge pull request #199 from nauttiilus/1.5.2-dev
Browse files Browse the repository at this point in the history 
fix: score calculation after de-allocation
  • Loading branch information
@bigsat309
bigsat309 authored Sep 26, 2024
2 parents d6eb91a + f7a66b6 commit f7d1a16
Show file tree
Hide file tree
Showing 5 changed files with 89 additions and 86 deletions.
6 changes: 3 additions & 3 deletions compute/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@
import string

# Define the version of the template module.
__version__ = "1.5.0"
__minimal_miner_version__ = "1.4.9"
__minimal_validator_version__ = "1.5.0"
__version__ = "1.5.2"
__minimal_miner_version__ = "1.5.1"
__minimal_validator_version__ = "1.5.2"

version_split = __version__.split(".")
__version_as_int__ = (100 * int(version_split[0])) + (10 * int(version_split[1])) + (1 * int(version_split[2]))
Expand Down
10 changes: 5 additions & 5 deletions neurons/Validator/calculate_pow_score.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,11 @@ def calc_score(response, hotkey, allocated_hotkeys, penalized_hotkeys, validator
:return:
"""
try:
challenge_attempts = prevent_none(response["challenge_attempts"])
challenge_successes = prevent_none(response["challenge_successes"])
last_20_challenge_failed = prevent_none(response["last_20_challenge_failed"])
challenge_elapsed_time_avg = prevent_none(response["challenge_elapsed_time_avg"])
challenge_difficulty_avg = prevent_none(response["last_20_difficulty_avg"])
challenge_attempts = prevent_none(response.get("challenge_attempts",1))
challenge_successes = prevent_none(response.get("challenge_successes",0))
last_20_challenge_failed = prevent_none(response.get("last_20_challenge_failed",0))
challenge_elapsed_time_avg = prevent_none(response.get("challenge_elapsed_time_avg", compute.pow_timeout))
challenge_difficulty_avg = prevent_none(response.get("last_20_difficulty_avg", compute.pow_min_difficulty))
has_docker = response.get("has_docker", False)

# Define base weights for the PoW
Expand Down
101 changes: 53 additions & 48 deletions neurons/Validator/database/challenge.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,56 +37,61 @@ def select_challenge_stats(db: ComputeDb) -> dict:
}
"""
cursor = db.get_cursor()

cursor.execute(
"""
WITH RankedChallenges AS (SELECT uid,
ss58_address,
success,
created_at,
ROW_NUMBER() OVER (PARTITION BY uid, ss58_address ORDER BY created_at DESC) AS row_num
FROM challenge_details)
SELECT main_query.uid,
main_query.ss58_address,
main_query.challenge_attempts,
main_query.challenge_successes,
main_query.challenge_elapsed_time_avg,
main_query.challenge_difficulty_avg,
COALESCE(main_query.challenge_failed, 0) as challenge_failed,
COALESCE(last_20_challenge_failed.last_20_challenge_failed, 0) as last_20_challenge_failed,
last_20_query.last_20_difficulty_avg
FROM (SELECT uid,
ss58_address,
COUNT(*) AS challenge_attempts,
COUNT(CASE WHEN success = 0 THEN 1 END) AS challenge_failed,
SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END) AS challenge_successes,
AVG(CASE WHEN success = 1 THEN elapsed_time END) AS challenge_elapsed_time_avg,
AVG(CASE WHEN success = 1 THEN difficulty END) AS challenge_difficulty_avg
FROM challenge_details
WHERE date(created_at) >= date('now', '-12 hours')
GROUP BY uid, ss58_address) AS main_query
LEFT JOIN (SELECT uid,
ss58_address,
COUNT(*) AS last_20_challenge_failed
FROM (SELECT uid, ss58_address, success
FROM RankedChallenges
WHERE row_num <= 20
ORDER BY created_at DESC) AS Last20Rows
WHERE success = 0
GROUP BY uid, ss58_address) AS last_20_challenge_failed
WITH RankedChallenges AS (
SELECT uid,
ss58_address,
success,
elapsed_time,
difficulty,
created_at,
ROW_NUMBER() OVER (PARTITION BY uid ORDER BY created_at DESC) AS row_num
FROM challenge_details
),
FilteredChallenges AS (
SELECT *
FROM RankedChallenges
WHERE row_num <= 60
)
SELECT main_query.uid,
main_query.ss58_address,
COUNT(*) AS challenge_attempts,
COUNT(CASE WHEN success = 0 THEN 1 END) AS challenge_failed,
SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END) AS challenge_successes,
AVG(CASE WHEN success = 1 THEN elapsed_time END) AS challenge_elapsed_time_avg,
AVG(CASE WHEN success = 1 THEN difficulty END) AS challenge_difficulty_avg,
COALESCE(last_20_challenge_failed.last_20_challenge_failed, 0) as last_20_challenge_failed,
last_20_query.last_20_difficulty_avg
FROM (SELECT *
FROM FilteredChallenges
ORDER BY created_at DESC) AS main_query
LEFT JOIN (SELECT uid,
ss58_address,
COUNT(*) AS last_20_challenge_failed
FROM (SELECT uid,
ss58_address,
success,
ROW_NUMBER() OVER (PARTITION BY uid ORDER BY created_at DESC) AS row_num
FROM challenge_details)
WHERE row_num <= 20 AND success = 0
GROUP BY uid, ss58_address) AS last_20_challenge_failed
ON main_query.uid = last_20_challenge_failed.uid AND
main_query.ss58_address = last_20_challenge_failed.ss58_address
LEFT JOIN (SELECT uid,
ss58_address,
AVG(difficulty) AS last_20_difficulty_avg
FROM (SELECT uid,
ss58_address,
difficulty,
ROW_NUMBER() OVER (PARTITION BY uid, ss58_address ORDER BY created_at DESC) AS row_num
FROM challenge_details
WHERE success = 1) AS subquery
WHERE row_num <= 20
GROUP BY uid, ss58_address) AS last_20_query
ON main_query.uid = last_20_query.uid AND main_query.ss58_address = last_20_query.ss58_address;
LEFT JOIN (SELECT uid,
ss58_address,
AVG(difficulty) AS last_20_difficulty_avg
FROM (SELECT uid,
ss58_address,
difficulty,
ROW_NUMBER() OVER (PARTITION BY uid ORDER BY created_at DESC) AS row_num
FROM challenge_details
WHERE success = 1)
WHERE row_num <= 20
GROUP BY uid, ss58_address) AS last_20_query
ON main_query.uid = last_20_query.uid AND main_query.ss58_address = last_20_query.ss58_address
GROUP BY main_query.uid, main_query.ss58_address;
"""
)

Expand All @@ -98,18 +103,18 @@ def select_challenge_stats(db: ComputeDb) -> dict:
uid,
ss58_address,
challenge_attempts,
challenge_failed,
challenge_successes,
challenge_elapsed_time_avg,
challenge_difficulty_avg,
challenge_failed,
last_20_challenge_failed,
last_20_difficulty_avg,
) = result
stats[uid] = {
"ss58_address": ss58_address,
"challenge_attempts": challenge_attempts,
"challenge_successes": challenge_successes,
"challenge_failed": int(challenge_failed) if challenge_failed else 0,
"challenge_successes": challenge_successes,
"challenge_elapsed_time_avg": challenge_elapsed_time_avg,
"challenge_difficulty_avg": challenge_difficulty_avg,
"last_20_challenge_failed": last_20_challenge_failed,
Expand Down
24 changes: 12 additions & 12 deletions neurons/miner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -656,18 +656,18 @@ async def start(self):
self.sync_status()

# Check port open
port = int(self.config.ssh.port)
if port:
result = check_port('localhost', port)
if result is True:
bt.logging.info(f"API: Port {port} on the server is open")
elif result is False:
bt.logging.info(f"API: Port {port} on the server is closed")
else:
bt.logging.warning(f"API: Could not determine status of port {port} on the server")
else:
bt.logging.warning(f"API: Could not find the server port that was provided to validator")
self.wandb.update_miner_port_open(result)
# port = int(self.config.ssh.port)
# if port:
# result = check_port('localhost', port)
# if result is True:
# bt.logging.info(f"API: Port {port} on the server is open")
# elif result is False:
# bt.logging.info(f"API: Port {port} on the server is closed")
# else:
# bt.logging.warning(f"API: Could not determine status of port {port} on the server")
# else:
# bt.logging.warning(f"API: Could not find the server port that was provided to validator")
# self.wandb.update_miner_port_open(result)

# check allocation status
self.__check_alloaction_errors()
Expand Down
34 changes: 16 additions & 18 deletions neurons/validator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -292,34 +292,32 @@ def sync_scores(self):

self.pretty_print_dict_values(self.stats)

self._queryable_uids = self.get_queryable()

# Calculate score
for uid in self.uids:
try:
# Determine if the user's hotkey has Docker
hotkey = self.stats[uid].get("ss58_address")
axon = self._queryable_uids[uid]
hotkey = axon.hotkey

if uid not in self.stats:
self.stats[uid] = {} # Initialize empty dictionary for this UID

if hotkey in has_docker_hotkeys:
self.stats[uid]["has_docker"] = True
elif not self.finalized_specs_once:
self.stats[uid]["has_docker"] = True
else:
self.stats[uid]["has_docker"] = False

# Find the maximum score of all uids excluding allocated uids
# max_score_uids = max(
# self.stats[uid]["score"]
# for uid in self.stats
# if "score" in self.stats[uid] and self.stats[uid].get("ss58_address") not in self.allocated_hotkeys
# )

# score = calc_score(self.stats[uid], hotkey=hotkey, allocated_hotkeys=self.allocated_hotkeys, max_score_uid=max_score_uids)
score = calc_score(self.stats[uid],
hotkey = hotkey,
allocated_hotkeys = self.allocated_hotkeys,
penalized_hotkeys = self.penalized_hotkeys,
validator_hotkeys = valid_validator_hotkeys)

score = calc_score(self.stats[uid], hotkey, self.allocated_hotkeys, self.penalized_hotkeys, valid_validator_hotkeys)
self.stats[uid]["score"] = score
except (ValueError, KeyError):

except KeyError as e:
# bt.logging.info(f"KeyError occurred for UID {uid}: {str(e)}")
score = 0
except Exception as e:
# bt.logging.info(f"An unexpected exception occurred for UID {uid}: {str(e)}")
score = 0

self.scores[uid] = score
Expand Down Expand Up @@ -858,7 +856,7 @@ async def start(self):
# Filter axons with stake and ip address.
self._queryable_uids = self.get_queryable()

self.sync_checklist()
#self.sync_checklist()

if self.current_block % block_next_sync_status == 0 or block_next_sync_status < self.current_block:
block_next_sync_status = self.current_block + 25 # ~ every 5 minutes
Expand Down

0 comments on commit f7d1a16

Please sign in to comment.