Tokenizer rules from the core spacy

fi/fi.py

@@ -1,13 +1,11 @@
 from .lemmatizer import FinnishLemmatizer
+from .punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES
 from spacy.lang.fi import FinnishDefaults
 from spacy.language import Language
 from spacy.lookups import Lookups
 from spacy.symbols import POS, PUNCT, SYM, ADJ, CCONJ, SCONJ, NUM, DET, ADV
 from spacy.symbols import ADP, X, VERB, NOUN, PROPN, PART, INTJ, SPACE, PRON, AUX
 
-# Punctuation stolen from Danish
-from spacy.lang.da.punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES
-
 
 TAG_MAP = {
     'Adv': {POS: ADV},

fi/punctuation.py

@@ -0,0 +1,33 @@
+# coding: utf8
+from __future__ import unicode_literals
+
+from spacy.lang.char_classes import LIST_ELLIPSES, LIST_ICONS
+from spacy.lang.char_classes import CONCAT_QUOTES, ALPHA, ALPHA_LOWER, ALPHA_UPPER
+from spacy.lang.punctuation import TOKENIZER_SUFFIXES
+
+
+_quotes = CONCAT_QUOTES.replace("'", "")
+
+_infixes = (
+    LIST_ELLIPSES
+    + LIST_ICONS
+    + [
+        r"(?<=[{al}])\.(?=[{au}])".format(al=ALPHA_LOWER, au=ALPHA_UPPER),
+        r"(?<=[{a}])[,!?](?=[{a}])".format(a=ALPHA),
+        r"(?<=[{a}])[:<>=](?=[{a}])".format(a=ALPHA),
+        r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA),
+        r"(?<=[{a}])([{q}\)\]\(\[])(?=[{a}])".format(a=ALPHA, q=_quotes),
+        r"(?<=[{a}])--(?=[{a}])".format(a=ALPHA),
+        r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA),
+    ]
+)
+
+_suffixes = [
+    suffix
+    for suffix in TOKENIZER_SUFFIXES
+    if suffix not in ["'s", "'S", "’s", "’S", r"\'"]
+]
+
+
+TOKENIZER_INFIXES = _infixes
+TOKENIZER_SUFFIXES = _suffixes

tools/package_model.sh

@@ -14,7 +14,7 @@ MODEL_NAME=`echo $PACKAGE_DIR | sed 's/-[0-9.]\+$//'`
 
 echo "Copying the lemmatizer sources to the package directory"
 mkdir -p models/python-package/"$PACKAGE_DIR/$MODEL_NAME"
-cp fi/fi.py fi/lemmatizer.py models/python-package/"$PACKAGE_DIR/$MODEL_NAME"/
+cp fi/fi.py fi/lemmatizer.py fi/punctuation.py models/python-package/"$PACKAGE_DIR/$MODEL_NAME"/
 cp -r fi/lookups/ models/python-package/"$PACKAGE_DIR/$MODEL_NAME"/
 
 echo "Adding import to __init__.py"