Add models (#179)

allenai · Sep 13, 2024 · 82bd441 · 82bd441
1 parent 6552e93
commit 82bd441
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 1 deletion.
diff --git a/scripts/configs/eval_configs.yaml b/scripts/configs/eval_configs.yaml
@@ -734,4 +734,29 @@ Skywork/Skywork-Reward-Llama-3.1-8B:
   batch_size: 8
   dpo: False
   torch_dtype: bfloat16
-  trust_remote_code: False
+  trust_remote_code: False
+LxzGordon/URM-LLaMa-3.1-8B:
+  model: LxzGordon/URM-LLaMa-3.1-8B
+  tokenizer: LxzGordon/URM-LLaMa-3.1-8B
+  chat_template: # none for tokenizer
+  batch_size: 4
+  dpo: False
+  trust_remote_code: True
+  quantized: False
+LxzGordon/URM-LLaMa-3-8B:
+  model: LxzGordon/URM-LLaMa-3-8B
+  tokenizer: LxzGordon/URM-LLaMa-3-8B
+  chat_template: # none for tokenizer
+  batch_size: 4
+  dpo: False
+  trust_remote_code: True
+  quantized: False
+# Skywork/Skywork-Critic-Llama-3.1-8B:
+#   model: Skywork/Skywork-Critic-Llama-3.1-8B
+#   tokenizer: Skywork/Skywork-Critic-Llama-3.1-8B
+#   chat_template: # none for tokenizer
+#   batch_size: 4
+#   dpo: False
+#   generative: True
+#   num_gpus: 1
+
diff --git a/scripts/submit_eval_jobs.py b/scripts/submit_eval_jobs.py
@@ -158,6 +158,9 @@
         d["tasks"][0]["arguments"][0] += " --pref_sets"
     if eval_bfloat16:
         d["tasks"][0]["arguments"][0] += " --torch_dtype=bfloat16"
+    if model_config["quantized"] is not None:
+        if not model_config["quantized"] and not eval_dpo:
+            d["tasks"][0]["arguments"][0] += " --not_quantized"
 
     # for run_rm only, for now, and gemma-2-27b RMs
     if "attention_implementation" in model_config: