fix: add tokenizer for the query string

lunr-repro.tsx

@@ -93,19 +93,30 @@ const Search: React.FC<{
   query: string;
   docs: Record<string, string>;
 }> = function ({ query, docs }) {
-  const lunrIndex = useMemo((() => lunr(function () {
-    this.ref('name');
-    this.field('body');
-    for (const [name, body] of Object.entries(docs)) {
-      this.use((lunr as any).ja);
-      this.add({ name, body });
-    }
-    //console.debug(`Indexed ${docs.length} docs`);
-  })), [docs]);
+  const lunrIndex = useMemo(
+    () =>
+      lunr(function () {
+        this.use((lunr as any).ja);
+        this.ref("name");
+        this.field("body");
+        for (const [name, body] of Object.entries(docs)) {
+          this.add({ name, body });
+        }
+        //console.debug(`Indexed ${docs.length} docs`);
+      }),
+    [docs]
+  );
 
   const [results, error] = useMemo(() => {
+    // example input = 標準製品
+    // in Japanese these are two words, 標準 and 製品. Which will be tokenized in the docs as such (separated).
+    // in order to search for 標準製品. Use the same tokenizer, to separate the query.
+    // create a tokenized query, which should use the function to tokenize the doc
+    const queryTokenized = lunr.ja.tokenizer(query);
+    // turn the array back into proper lunr query search term. such as "標準 製品"
+    const queryTerm = queryTokenized.join(" ");
     try {
-      return [lunrIndex.search(query) ?? [], null];
+      return [lunrIndex.search(queryTerm) ?? [], null];
     } catch (e) {
       return [[], `${e.message}`];
     }