Landedb (#21)

* make disable experimentalwarning work

Signed-off-by: cbh778899 <cbh778899@outlook.com>

* add lancedb related depedencies

Signed-off-by: cbh778899 <cbh778899@outlook.com>

* add URL in RequestOptions & fix port issues

Signed-off-by: cbh778899 <cbh778899@outlook.com>

* implement basic lancedb and dataset

Signed-off-by: cbh778899 <cbh778899@outlook.com>

* rename eng option rag to embedding

Signed-off-by: cbh778899 <cbh778899@outlook.com>

* update the database route to make sense

Signed-off-by: cbh778899 <cbh778899@outlook.com>

---------

Signed-off-by: cbh778899 <cbh778899@outlook.com>

database/index.js

@@ -0,0 +1,42 @@
+import * as lancedb from "@lancedb/lancedb";
+import { get, post } from "../tools/request.js"
+import { Schema, Field, FixedSizeList, Int16, Float16, Utf8 } from "apache-arrow";
+
+const uri = "/tmp/lancedb/";
+const db = await lancedb.connect(uri);
+
+const table = await db.createEmptyTable("rag_data", new Schema([
+    new Field("id", new Int16()),
+    new Field("vector", new FixedSizeList(384, new Field("item", new Float16(), true)), false),
+    new Field("question", new Utf8()),
+    new Field("answer", new Utf8())
+]), {
+    // mode: "overwrite",
+    existOk: true
+})
+
+export async function loadDataset(dataset_link) {
+    const {rows, http_error} = await get('', {}, { URL: dataset_link })
+    if(http_error) {
+        return false;
+    }
+    await table.add(rows.map(({ row_id, row })=>{
+        const { question, answer, question_embedding } = row;
+        return { id: row_id, question, answer, vector: question_embedding }
+    }))
+    return true;
+}
+
+export async function searchByEmbedding(vector) {
+    const record = await table.search(vector).limit(1).toArray();
+    if(!record.length) return null;
+    const { question, answer } = record[0];
+    return { question, answer };
+}
+
+export async function searchByMessage(msg) {
+    const { embedding } = await post('embedding', {body: {
+        content: msg
+    }}, { eng: "embedding" });
+    return await searchByEmbedding(embedding);
+}

package.json

@@ -5,8 +5,8 @@
   "description": "OpenAI like APIs",
   "main": "index.js",
   "scripts": {
-    "start": "node index.js --disable-warning=ExperimentalWarning",
-    "dev": "nodemon index.js --disable-warning=ExperimentalWarning",
+    "start": "node --disable-warning=ExperimentalWarning index.js",
+    "dev": "nodemon --disable-warning=ExperimentalWarning index.js",
     "lint": "npx eslint .",
     "build": "npm install && node index.js"
   },
@@ -17,6 +17,8 @@
     "@babel/eslint-parser": "^7.25.1",
     "@babel/plugin-syntax-import-attributes": "^7.24.7",
     "@eslint/js": "^9.8.0",
+    "@lancedb/lancedb": "^0.8.0",
+    "apache-arrow": "^17.0.0",
     "body-parser": "^1.20.2",
     "cors": "^2.8.5",
     "dotenv": "^16.4.5",