fix 2:4 examples (vllm-project#161)

dbarbuzzi · Sep 10, 2024 · f8e0ef6 · f8e0ef6
1 parent fa21e45
commit f8e0ef6
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 5 deletions.
diff --git a/examples/quantization_24_sparse_w4a16/2:4_w4a16_group-128_recipe.yaml b/examples/quantization_24_sparse_w4a16/2:4_w4a16_group-128_recipe.yaml
@@ -23,12 +23,14 @@ quantization_stage:
   run_type: oneshot
   quantization_modifiers:
     GPTQModifier:
-      sequential_update: false
+      sequential_update: true
+      ignore: ["lm_head"]
       config_groups:
         group_0:
           weights:
             num_bits: 4
             type: "int"
             symmetric: true
-            strategy: "channel"
-          targets: ["Linear"]
+            strategy: "group"
+            group_size: 128
+          targets: ["Linear"]
diff --git a/examples/quantization_24_sparse_w4a16/2:4_w4a16_recipe.yaml b/examples/quantization_24_sparse_w4a16/2:4_w4a16_recipe.yaml
@@ -23,7 +23,7 @@ quantization_stage:
   run_type: oneshot
   quantization_modifiers:
     GPTQModifier:
-      sequential_update: false
+      sequential_update: true
       ignore: ["lm_head"]
       config_groups:
         group_0:

diff --git a/examples/quantization_24_sparse_w4a16/llama7b_sparse_w4a16.py b/examples/quantization_24_sparse_w4a16/llama7b_sparse_w4a16.py
@@ -23,7 +23,7 @@
 num_calibration_samples = 512
 
 # set training parameters for finetuning
-num_train_epochs = 0.5
+num_train_epochs = 0.01
 logging_steps = 500
 save_steps = 5000
 gradient_checkpointing = True  # saves memory during training