fix deberta implementation

adapter-hub · Jan 7, 2025 · dee1023 · dee1023
1 parent c3ab05a
commit dee1023
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/src/adapters/models/deberta/modeling_deberta.py b/src/adapters/models/deberta/modeling_deberta.py
@@ -144,7 +144,11 @@ def forward(
             # >>> END AH Changes <<<
 
         if rel_att is not None:
-            attention_scores = attention_scores + rel_att
+            # >>> START AH Changes <<<
+            rel_att_padded = torch.zeros_like(attention_scores)
+            rel_att_padded[:, :, :, -rel_att.size(-1) :] = rel_att
+            # >>> END AH Changes <<<
+            attention_scores = attention_scores + rel_att_padded
 
         # bxhxlxd
         if self.head_logits_proj is not None: