diff --git a/recipes/configs/qwen2_5/evaluation.yaml b/recipes/configs/qwen2_5/evaluation.yaml
new file mode 100644
index 000000000..e3cf6169e
--- /dev/null
+++ b/recipes/configs/qwen2_5/evaluation.yaml
@@ -0,0 +1,41 @@
+# Config for EleutherEvalRecipe in eleuther_eval.py
+#
+# To launch, run the following command from root torchtune directory:
+#    tune run eleuther_eval --config eleuther_evaluation tasks=["truthfulqa_mc2","hellaswag"]
+
+output_dir: ./ # Not needed
+
+# Model Arguments
+model:
+  _component_: torchtune.models.qwen2_5.qwen2_5_0_5b
+
+checkpointer:
+  _component_: torchtune.training.FullModelHFCheckpointer
+  checkpoint_dir: /tmp/Qwen2_5-0_5B-Instruct
+  checkpoint_files: [
+    model.safetensors,
+  ]
+  output_dir: ${output_dir}
+  model_type: QWEN2
+
+# Tokenizer
+tokenizer:
+  _component_: torchtune.models.qwen2_5.qwen2_5_tokenizer
+  path: /tmp/Qwen2_5-0_5B-Instruct/vocab.json
+  merges_file: /tmp/Qwen2_5-0_5B-Instruct/merges.txt
+  max_seq_len: null
+
+# Environment
+device: cuda
+dtype: bf16
+seed: 1234 # It is not recommended to change this seed, b/c it matches EleutherAI's default seed
+
+# EleutherAI specific eval args
+tasks: ["truthfulqa_mc2"]
+limit: null
+max_seq_length: 4096
+batch_size: 8
+enable_kv_cache: True
+
+# Quantization specific args
+quantizer: null
diff --git a/torchtune/_recipe_registry.py b/torchtune/_recipe_registry.py
index 8282f33da..4b504315a 100644
--- a/torchtune/_recipe_registry.py
+++ b/torchtune/_recipe_registry.py
@@ -457,6 +457,10 @@ class Recipe:
                 name="qwen2/evaluation",
                 file_path="qwen2/evaluation.yaml",
             ),
+            Config(
+                name="qwen2_5/evaluation",
+                file_path="qwen2_5/evaluation.yaml",
+            ),
             Config(
                 name="gemma/evaluation",
                 file_path="gemma/evaluation.yaml",