correctly dropout positions

lucidrains · Oct 31, 2021 · 923522d · 923522d
1 parent d6c7c92
commit 923522d
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 1 deletion.
diff --git a/enformer_pytorch/enformer_pytorch.py b/enformer_pytorch/enformer_pytorch.py
@@ -212,6 +212,7 @@ def forward(self, x):
         content_logits = einsum('b h i d, b h j d -> b h i j', q + self.rel_content_bias, k)
 
         positions = get_positional_embed(n, self.num_rel_pos_features, device)
+        positions = self.pos_dropout(positions)
         rel_k = self.to_rel_k(positions)
 
         rel_k = rearrange(rel_k, 'n (h d) -> h n d', h = h)

diff --git a/setup.py b/setup.py
@@ -3,7 +3,7 @@
 setup(
   name = 'enformer-pytorch',
   packages = find_packages(exclude=[]),
-  version = '0.0.9',
+  version = '0.0.10',
   license='MIT',
   description = 'Enformer - Pytorch',
   author = 'Phil Wang',