Merge pull request #320 from macrocosm-os/staging

Staging

folding/__init__.py

@@ -1,7 +1,7 @@
 from .protocol import JobSubmissionSynapse
 from .validators.protein import Protein
 
-__version__ = "1.4.1"
+__version__ = "1.4.2"
 version_split = __version__.split(".")
 __spec_version__ = (10000 * int(version_split[0])) + (100 * int(version_split[1])) + (1 * int(version_split[2]))
 

folding/base/validator.py

@@ -220,7 +220,6 @@ async def update_scores(self, rewards: torch.FloatTensor, uids: List[int]):
             # Replace any NaN values in rewards with 0.
             rewards = torch.nan_to_num(rewards, 0)
 
-        # Check if `uids` is already a tensor and clone it to avoid the warning.
         if isinstance(uids, torch.Tensor):
             uids_tensor = uids.clone().detach()
         else:
@@ -239,6 +238,13 @@ async def update_scores(self, rewards: torch.FloatTensor, uids: List[int]):
         self.scores: torch.FloatTensor = alpha * scattered_rewards + (
             1 - alpha
         ) * self.scores.to(self.device)
+
+        # Suppress scores for malicious hotkeys if they exist. Temp.
+        mask = np.isin(self.metagraph.hotkeys, self.malicious_hotkeys)
+        uids_to_suppress = self.metagraph.uids[mask]
+        if len(uids_to_suppress) > 0:
+            self.scores[uids_to_suppress] = 0
+
         logger.debug(f"Updated moving avg scores: {self.scores}")
 
     def save_state(self):

folding/utils/config.py

@@ -73,7 +73,7 @@ def add_args(cls, parser):
         "--neuron.epoch_length",
         type=int,
         help="The default epoch length (how often we set weights, measured in 12 second blocks).",
-        default=250,
+        default=150,
     )
 
     parser.add_argument(
@@ -326,7 +326,7 @@ def add_validator_args(cls, parser):
         "--neuron.update_interval",
         type=float,
         help="The interval in which the validators query the miners for updates. (seconds)",
-        default=300,  # samples every 5-minutes in the simulation.
+        default=60,  # samples every 5-minutes in the simulation.
     )
 
     parser.add_argument(

folding/validators/reward.py

@@ -1,13 +1,40 @@
 import time
 from typing import List
+from itertools import chain
+from collections import defaultdict 
 
-import bittensor as bt
 import numpy as np
 
 from folding.protocol import JobSubmissionSynapse
 from folding.validators.protein import Protein
 from folding.utils.logger import logger
 
+def check_if_identical(event):
+    """ method to check if any of the submissions are idential. If they are, 0 reward for any uids involved.
+    This should only happen for uids that are submitting the same results from a single owner. 
+
+    We look at the simulated energy via the validator to avoid tampering. Anything that is not fully reprod will be 
+    caught inside of the protein.is_run_valid method.
+    """
+
+    groups = defaultdict(list)
+    for idx, energy_list in enumerate(event["checked_energy"]):
+        if energy_list == 0:
+            continue
+
+        groups[tuple(energy_list)].append(idx)
+
+    # Display identical groups
+    identical_groups = [indices for indices in groups.values() if len(indices) > 1]
+    flattened_list = list(chain.from_iterable(identical_groups))
+
+    if len(flattened_list) > 0:
+        logger.warning(f"Setting {len(flattened_list)} / {len(event['checked_energy'])} uids to 0 reward due to identical submissions.")
+        for idx in flattened_list:
+            event["is_valid"][idx] = False
+            event["reason"][idx] = "Identical submission to another hotkey in the group"
+
+    return event 
 
 def get_energies(protein: Protein, responses: List[JobSubmissionSynapse], uids: List[int]):
     """Takes all the data from reponse synapses, checks if the data is valid, and returns the energies.
@@ -66,8 +93,6 @@ def get_energies(protein: Protein, responses: List[JobSubmissionSynapse], uids:
             event["is_run_valid_time"][i] = time.time() - start_time
             event["reason"][i] = reason
 
-            energies[i] = np.median(checked_energy[-10:]) if is_valid else 0
-
             event["checked_energy"][i] = checked_energy
             event["miner_energy"][i] = miner_energy
             event["is_valid"][i] = is_valid
@@ -80,4 +105,13 @@ def get_energies(protein: Protein, responses: List[JobSubmissionSynapse], uids:
             logger.error(f"Failed to parse miner data for uid {uid} with error: {E}")
             continue
 
+    # Check if the miners return identical energy results. 
+    event = check_if_identical(event)
+
+    for idx, is_valid in enumerate(event["is_valid"]):
+        if is_valid: 
+            energies[idx] = np.median(event["checked_energy"][idx][-10:]) #energy that we computed... 
+        else:
+            energies[idx] = 0
+
     return energies, event

neurons/validator.py

@@ -50,7 +50,7 @@ def __init__(self, config=None):
         # Sample all the uids on the network, and return only the uids that are non-valis.
         logger.info("Determining all miner uids...⏳")
         self.all_miner_uids: List = get_random_uids(self, k=int(self.metagraph.n), exclude=None).tolist()
-        self.all_miner_uids: List = self.filter_miners()
+        self.all_miner_uids, self.malicious_hotkeys = self.filter_miners()
 
         self.wandb_run_start = None
         self.RSYNC_EXCEPTION_COUNT = 0
@@ -69,6 +69,7 @@ def filter_miners(self) -> List[int]:
 
         logger.warning(f"Blacklist on the following coldkeys: {blacklist}")
         malicious_uids = np.where(np.isin(self.metagraph.coldkeys, blacklist))[0]
+        malicious_hotkeys = [self.metagraph.hotkeys[uid] for uid in malicious_uids]
 
         # Remove the malicious uids from the list of all miner uids
         logger.warning(f"Removing {len(malicious_uids)} malicious uids from the list of all miner uids.")
@@ -77,7 +78,7 @@ def filter_miners(self) -> List[int]:
         all_miner_uids = list(np.array(self.all_miner_uids)[mask])
         logger.info(f"Initial number of uids: {len(self.all_miner_uids)}. After filter: {len(all_miner_uids)}")
 
-        return all_miner_uids
+        return all_miner_uids, malicious_hotkeys
 
     def parse_mdrun_args(self) -> str:
         mdrun_args = ""
@@ -304,6 +305,18 @@ async def update_job(self, job: Job):
         top_reward = 0.80
         apply_pipeline = False
 
+        # In the situation where there are entities that are malicious. 
+        mask = np.isin(self.metagraph.hotkeys, self.malicious_hotkeys)
+        uids_to_suppress = self.metagraph.uids[mask]
+        suppressed_uids = []
+        for idx, uid in enumerate(job.event["uids"]):
+            if uid in uids_to_suppress:
+                suppressed_uids.append(uid)
+                job.event["energies"][idx] = 0 
+
+        if len(suppressed_uids) > 0: 
+            logger.warning(f"Suppressed uids: {suppressed_uids} due to malicious behaviour")
+
         # There could be hotkeys that have decided to stop serving. We need to remove them from the store.
         serving_hotkeys = []
         for ii, state in enumerate(job.event["response_miners_serving"]):

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "folding"
-version = "1.4.1"
+version = "1.4.2"
 description = "Macrocosmos Subnet 25: Folding"
 authors = ["Brian McCrindle <[email protected]>", "Sergio Champoux <[email protected]>", "Szymon Fonau <[email protected]>"]