API for /v1/files (#101)

* Add tokens count for chat completion

Signed-off-by: NguyenNguyen205 <s3927220@rmit.edu.vn>

* Set up files APIs

Signed-off-by: NguyenNguyen205 <s3927220@rmit.edu.vn>

* Add check file format on endpoint

Signed-off-by: NguyenNguyen205 <s3927220@rmit.edu.vn>

* Add get all endpoints, save data to lance db, modify npm run dev

Signed-off-by: NguyenNguyen205 <s3927220@rmit.edu.vn>

* Add documentation for 2 endpoints

Signed-off-by: NguyenNguyen205 <s3927220@rmit.edu.vn>

* Add code commenting and fix small bugs

Signed-off-by: NguyenNguyen205 <s3927220@rmit.edu.vn>

---------

Signed-off-by: NguyenNguyen205 <s3927220@rmit.edu.vn>

actions/file.js

@@ -0,0 +1,95 @@
+// coding=utf-8
+
+import { randomUUID } from "crypto";
+import { extractAPIKeyFromRequest, validateAPIKey } from "../tools/apiKey.js";
+import * as fs from 'fs';
+import { getAllFilesData, loadFileToDatabase } from "../database/file-handling.js";
+
+// Copyright [2024] [SkywardAI]
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//        http://www.apache.org/licenses/LICENSE-2.0
+
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * function for upload a file
+ * @param {Request} req 
+ * @param {Response} res 
+ */
+
+export async function uploadFile(req, res) {
+    if (!validateAPIKey(extractAPIKeyFromRequest(req))) {
+        res.status(401).send("Not Authorized!");
+        return;
+    }
+    const { file } = req;
+    if (!file) {
+        res.status(400).send("Input file not specified");
+        return;
+    }
+
+    // Check file size limit (10MB)
+    if (file.size / 1000000 > 10) {
+        res.status(400).send("Only accepting file size smaller than 10MB");
+        return;
+    }
+
+    // Check file format
+    let acceptedFormat = "application/json"
+    if (acceptedFormat.localeCompare(file.mimetype) != 0) {
+        res.status(400).send("File format not supported");
+        return;
+    }
+
+    // Load in database
+    let resBody = {
+        "id": randomUUID(),
+        "bytes": file.size,
+        "created_at": Date.now(),
+        "filename": file.originalname,
+    }
+
+    const result = await loadFileToDatabase(resBody);
+    if (!result) {
+        res.status(500).send("Can't save to database");
+        return;
+    }
+
+    // load file
+    const uploadPath = `files/${file.originalname}`;
+    fs.writeFileSync(uploadPath, file.buffer, (err) => {
+        if (err) throw err;
+        console.log("File has been saved");
+    })
+
+    res.status(200).send(resBody);
+    return;
+}
+
+/**
+ * function for get all files metadata
+ * @param {Request} req 
+ * @param {Response} res 
+ */
+
+export async function getAllFiles(req, res) {
+    if (!validateAPIKey(extractAPIKeyFromRequest(req))) {
+        res.status(401).send("Not Authorized!");
+        return;
+    }
+
+    let resBody = await getAllFilesData();
+
+    const completeRes = {};
+    completeRes['data'] = resBody;
+
+    res.status(200).send(completeRes);
+    return;
+}

database/file-handling.js

@@ -0,0 +1,61 @@
+// coding=utf-8
+
+// Copyright [2024] [SkywardAI]
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//        http://www.apache.org/licenses/LICENSE-2.0
+
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { getTable } from "./index.js";
+import { FILE_TABLE } from "./types.js";
+
+/**
+ * @typedef FileMetadataStructure
+ * @property {String} id Id of the uploaded file
+ * @property {Number} bytes Size of the uploaded file
+ * @property {Number} created_at Date of uploaded, measure in miliseconds
+ * @property {String} filename Name of the file 
+ */
+
+/**
+ * Get all files metadata currently in database
+ * @returns {Promise<FileMetadataStructure[]>}
+ */
+export async function getAllFilesData() {
+    const file_table = await getTable(FILE_TABLE);
+
+    let queryResult = await file_table.query().toArray();
+    let result = []
+    for (let i in queryResult) {
+        const batch = queryResult[i];
+        const mid = {
+            id: batch.id,
+            bytes: batch.bytes,
+            created_at: Number(batch.created_at),
+            filename: batch.filename
+        }
+        result.push(mid)
+    }
+
+    return result;
+}
+
+/**
+ * Upload file metadata into database
+ * @param {FileMetadataStructure} fileData Metadata of the uploaded file
+ * @returns {Boolean} Success status of storing into database
+ */
+export async function loadFileToDatabase(fileData) {
+    const file_table = await getTable(FILE_TABLE);    
+
+    await file_table.add([{ id: fileData.id, bytes: fileData.bytes, created_at: fileData.created_at, filename: fileData.filename }])
+
+    return true;
+}

database/index.js

@@ -16,11 +16,11 @@
 import { connect } from "@lancedb/lancedb";
 import { 
     Schema, Field, FixedSizeList, 
-    Float32, Utf8, Int32,
+    Float32, Utf8, Int32, Int64,
     // eslint-disable-next-line
     Table 
 } from "apache-arrow";
-import { API_KEY_TABLE, DATASET_TABLE, SYSTEM_TABLE } from "./types.js";
+import { API_KEY_TABLE, DATASET_TABLE, FILE_TABLE, SYSTEM_TABLE } from "./types.js";
 
 const uri = "/tmp/lancedb/";
 const db = await connect(uri);
@@ -44,6 +44,13 @@ export async function initDB(force = false) {
         new Field("api_key", new Utf8()),
         new Field("usage", new Int32())
     ]), open_options);
+    // create or re-open file table
+    await db.createEmptyTable(FILE_TABLE, new Schema([
+        new Field("id", new Utf8()),
+        new Field("bytes", new Int32()),
+        new Field("created_at", new Int64()),
+        new Field("filename", new Utf8())
+    ]), open_options);
 }
 
 /**

database/types.js

@@ -15,4 +15,5 @@
 
 export const SYSTEM_TABLE = 'system';
 export const DATASET_TABLE = 'dataset';
-export const API_KEY_TABLE = 'api_tokens'
+export const API_KEY_TABLE = 'api_tokens';
+export const FILE_TABLE = 'files'

package.json

@@ -6,7 +6,7 @@
   "main": "index.js",
   "scripts": {
     "start": "node --disable-warning=ExperimentalWarning index.js",
-    "dev": "nodemon --disable-warning=ExperimentalWarning index.js",
+    "dev": "nodemon --disable-warning=ExperimentalWarning index.js --ignore files",
     "lint": "npx eslint .",
     "build": "npm install && node index.js"
   },
@@ -25,8 +25,10 @@
     "eslint": "^9.8.0",
     "express": "^4.19.2",
     "globals": "^15.8.0",
+    "multer": "^1.4.5-lts.1",
+    "nodemon": "^3.1.7",
     "prom-client": "12",
     "swagger-stats": "^0.99.7",
     "swagger-ui-express": "^5.0.1"
   }
-}
+}

routes/file.js

@@ -0,0 +1,13 @@
+import { Router } from "express";
+import { getAllFiles, uploadFile } from "../actions/file.js";
+import multer from "multer";
+
+export default function fileRoute() {
+    const router = Router();
+    const upload = multer();
+
+    router.post('', upload.single('input'), uploadFile);
+    router.get('', getAllFiles);
+
+    return router;
+}

routes/index.js

@@ -24,6 +24,7 @@ import embeddingRoute from "./embedding.js";
 import versionRoute from "./version.js";
 import { isRouteEnabled } from "../tools/enabledApiDecoder.js";
 import { generateScript } from "../tools/web_embed.js";
+import fileRoute from "./file.js";
 
 function indexRoute() {
     const router = Router();
@@ -55,6 +56,7 @@ function generateAPIRouters() {
     // api_router.use('/encoder', encoderRoute());
     // api_router.use('/decoder', decoderRoute());
     isRouteEnabled("version") && api_router.use('/version', versionRoute());
+    isRouteEnabled("file") && api_router.use('/files', fileRoute());
 
     return api_router;
 }