automatically add tokenize,ssplit

stanfordnlp · Jan 20, 2022 · 7a24b58 · 7a24b58
1 parent f44e693
commit 7a24b58
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/src/edu/stanford/nlp/pipeline/StanfordCoreNLP.java b/src/edu/stanford/nlp/pipeline/StanfordCoreNLP.java
@@ -245,6 +245,10 @@ public StanfordCoreNLP(Properties props, boolean enforceRequirements, AnnotatorP
         logger.info("preTokenized option set: Changing annotators tokenize,ssplit,mwt to tokenize,ssplit");
       } else if (oldAnnotators != null && oldAnnotators.startsWith("tokenize,ssplit")) {
         logger.info("preTokenized option set: Annotators list starts with tokenize,ssplit, no change needed.");
+      } else if (oldAnnotators != null && !oldAnnotators.contains("tokenize") && !oldAnnotators.contains("mwt")
+                 && !oldAnnotators.contains("ssplit") && !oldAnnotators.contains("cdc_tokenize")) {
+        logger.info("preTokenized option set: Adding tokenize,ssplit to beginning.");
+        newAnnotators = "tokenize,ssplit," + oldAnnotators;
       } else {
         logger.warn("preTokenized option set: Non-standard annotators list, preTokenized may not work in this case."); 
       }