Merge pull request #46 from opentensor/staging

2.0.2 Release

prompting/validators/__init__.py

@@ -27,7 +27,7 @@
 from . import event
 from . import dataset
 
-__version__ = "2.0.1"
+__version__ = "2.0.2"
 version_split = __version__.split(".")
 __spec_version__ = (
     (1000 * int(version_split[0]))

prompting/validators/forward.py

@@ -98,6 +98,22 @@ async def run_step(
         timeout=timeout,
     )
 
+    # Restrict the format of acceptable followup completions.
+    for response in responses:
+        # remove leading and trailing periods
+        completion = response.completion.strip(".")
+
+        if "followup" in name and len(completion) > 0:
+            if "?" in completion:
+                # take first question that is found and only use the sentence before the question mark
+                completion = completion.split("?")[0].split(".")[-1]
+            else:
+                # otherwise take the last sentence
+                completion = completion.split(".")[-1].split(".")[-1]
+
+            # take maximum of 40 words
+            response.completion = " ".join(completion.split(" ")[-40:]) + "?"
+
     # Compute the rewards for the responses given the prompt.
     rewards: torch.FloatTensor = torch.zeros(len(responses), dtype=torch.float32).to(
         self.device

prompting/validators/reward/config.py

@@ -38,8 +38,8 @@ class DefaultRewardFrameworkConfig:
     Note: All the weights should add up to 1.0.
     """
 
-    dpo_model_weight: float = 0.3
+    dpo_model_weight: float = 0.2
     rlhf_model_weight: float = 0.4
-    reciprocate_model_weight: float = 0.3
+    reciprocate_model_weight: float = 0.4
     dahoas_model_weight: float = 0
     prompt_model_weight: float = 0

prompting/validators/reward/diversity.py

@@ -68,6 +68,7 @@ def __init__(self, device: str):
         self.history_reward_bottom_k = 2
         self.historic_embeddings = torch.tensor([]).to(self.device)
         self.history_range = (500, 15500)
+        self.boundary = 0.5
 
     def get_embeddings(self, sentences: List[str]) -> "torch.FloatTensor":
         """Runs a forward pass through the model.
@@ -176,5 +177,7 @@ def get_rewards(
         else:
             return batch_rewards
 
-    def normalize_rewards(self, rewards: torch.FloatTensor) -> torch.FloatTensor:
+    def normalize_rewards(self, raw_rewards: torch.FloatTensor) -> torch.FloatTensor:
+        # Applies binarization on the rewards.
+        rewards = (raw_rewards > self.boundary).float()
         return rewards