diff --git a/scripts/configs/eval_configs.yaml b/scripts/configs/eval_configs.yaml index 7e8dfcb..2a359ec 100644 --- a/scripts/configs/eval_configs.yaml +++ b/scripts/configs/eval_configs.yaml @@ -734,4 +734,29 @@ Skywork/Skywork-Reward-Llama-3.1-8B: batch_size: 8 dpo: False torch_dtype: bfloat16 - trust_remote_code: False \ No newline at end of file + trust_remote_code: False +LxzGordon/URM-LLaMa-3.1-8B: + model: LxzGordon/URM-LLaMa-3.1-8B + tokenizer: LxzGordon/URM-LLaMa-3.1-8B + chat_template: # none for tokenizer + batch_size: 4 + dpo: False + trust_remote_code: True + quantized: False +LxzGordon/URM-LLaMa-3-8B: + model: LxzGordon/URM-LLaMa-3-8B + tokenizer: LxzGordon/URM-LLaMa-3-8B + chat_template: # none for tokenizer + batch_size: 4 + dpo: False + trust_remote_code: True + quantized: False +# Skywork/Skywork-Critic-Llama-3.1-8B: +# model: Skywork/Skywork-Critic-Llama-3.1-8B +# tokenizer: Skywork/Skywork-Critic-Llama-3.1-8B +# chat_template: # none for tokenizer +# batch_size: 4 +# dpo: False +# generative: True +# num_gpus: 1 + \ No newline at end of file diff --git a/scripts/submit_eval_jobs.py b/scripts/submit_eval_jobs.py index 493a187..ee371ee 100644 --- a/scripts/submit_eval_jobs.py +++ b/scripts/submit_eval_jobs.py @@ -158,6 +158,9 @@ d["tasks"][0]["arguments"][0] += " --pref_sets" if eval_bfloat16: d["tasks"][0]["arguments"][0] += " --torch_dtype=bfloat16" + if model_config["quantized"] is not None: + if not model_config["quantized"] and not eval_dpo: + d["tasks"][0]["arguments"][0] += " --not_quantized" # for run_rm only, for now, and gemma-2-27b RMs if "attention_implementation" in model_config: