Update delimiter list in preprocess_input_text function

pha123661 · Dec 11, 2023 · 2079ad1 · 2079ad1
1 parent de908d2
commit 2079ad1
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/app/app.py b/app/app.py
@@ -213,7 +213,7 @@ async def query(input_text):
 def preprocess_input_text(input_text: str):
     input_text = re.sub(r"https?://\S+|www\.\S+", "", input_text)
     input_text = input_text.strip(" ，。,.\n")
-    parts = re.split(r'(\s*[ ，。\n]\s*)', input_text)
+    parts = re.split(r'(\s*[ ，。？；,.\n]\s*)', input_text)
 
     text_list = parts[::2]
     delimiter_list = parts[1::2]