diff --git a/src/peft/tuners/lora/tp_layer.py b/src/peft/tuners/lora/tp_layer.py
index 676430cf38..d55731dce5 100644
--- a/src/peft/tuners/lora/tp_layer.py
+++ b/src/peft/tuners/lora/tp_layer.py
@@ -111,7 +111,7 @@ def update_layer(
         self.lora_B[adapter_name] = lora_b
         self.scaling[adapter_name] = lora_alpha / r
         if init_lora_weights:
-            self.reset_lora_parameters(adapter_name)
+            self.reset_lora_parameters(adapter_name, init_lora_weights)
 
         weight = getattr(self.get_base_layer(), "weight", None)
         if weight is not None:
diff --git a/tests/test_lora_megatron.py b/tests/test_lora_megatron.py
index 80d0f43010..4244dd9735 100644
--- a/tests/test_lora_megatron.py
+++ b/tests/test_lora_megatron.py
@@ -85,6 +85,7 @@ def __init__(self, config: TransformerConfig):
                 init_method=init.xavier_normal_,
                 bias=False,
                 input_is_parallel=True,
+                skip_bias_add=True,
             )
 
         def forward(self, input):