Bug in trie

dialectid/tests/test_text_repr.py

@@ -56,4 +56,13 @@ def test_SeqTM():
     assert seq.compute_tokens('~dias~duros~') == _
     assert seq.compute_tokens('~🤷~') == [['🤷']]
     assert seq.compute_tokens('~🙇🏿~') == [['🙇']]
-    assert seq.tokenize('buenos dias 🙇🏿')[-1] == '🙇'
+    assert seq.tokenize('buenos dias 🙇🏿')[-1] == '🙇'
+
+
+def test_SeqTM_bug():
+    """Test SeqTM class"""
+
+    seq = SeqTM(language='es', subwords=True, voc_size_exponent=13)
+    res1 = seq.tokenize('mira pinche a')
+    res2 = seq.tokenize('a pinche a')
+    assert res1[1:] == res2[1:]

dialectid/text_repr.py

@@ -139,6 +139,11 @@ def __vocabulary(self, counter):
             key = value
             if value[:2] == 'q:':
                 key = value[2:]
+                if key in self._map:
+                    continue
+                self._map[key] = value
+            else:
+                key = f'~{key}~'
                 self._map[key] = value
             tokens[key] = value
         _ = join(dirname(__file__), 'data', 'emojis.json.gz')
@@ -241,7 +246,7 @@ def find_token(self, text):
         :rtype: list
         """
 
-        blocks = list()
+        blocks = []
         init = i = end = 0
         head = self.data_structure
         current = head
@@ -257,12 +262,12 @@ def find_token(self, text):
                 current = head
                 if end > init:
                     blocks.append([init, end])
-                    if (end - init) > 2 and text[end - 1] == '~':
+                    if (end - init) >= 2 and text[end - 1] == '~':
                         init = i = end = end - 1
                     else:
                         init = i = end
                 elif i > init:
-                    if (i - init) > 2 and text[i - 1] == '~':
+                    if (i - init) >= 2 and text[i - 1] == '~':
                         init = end = i = i - 1
                     else:
                         init = end = i