feat: created vector db and embeddings system

.github/workflows/ci.yml

@@ -1,8 +1,5 @@
 name: Beavs AI Continuous Integration
 
-env:
-  HUSKY: 0
-
 on:
   push:
     branches:
@@ -14,6 +11,8 @@ on:
 jobs:
   ci:
     runs-on: ubuntu-latest
+    env:
+      HUSKY: 0
 
     steps:
       - name: Checkout code
@@ -36,5 +35,15 @@ jobs:
       - name: Type-check
         run: npm run type-check
 
+      - name: Create .env.local file
+        run: |
+          echo "PINECONE_API_KEY=${{ secrets.PINECONE_API_KEY }}" >> .env.local
+          echo "PINECONE_INDEX_NAME=${{ secrets.PINECONE_INDEX_NAME }}" >> .env.local
+          echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> .env.local
+
       - name: Build project
+        env:
+          PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
+          PINECONE_INDEX_NAME: ${{ secrets.PINECONE_INDEX_NAME }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         run: npm run build

package.json

@@ -2,6 +2,7 @@
     "name": "beavsai",
     "version": "0.1.0",
     "private": true,
+    "type": "module",
     "scripts": {
         "dev": "npm run db:generate && npm run db:migrate && next dev --turbopack",
         "build": "next build",
@@ -19,6 +20,7 @@
     "dependencies": {
         "@auth/prisma-adapter": "^2.7.2",
         "@hookform/resolvers": "^3.9.0",
+        "@pinecone-database/pinecone": "^4.0.0",
         "@prisma/client": "^5.21.1",
         "@radix-ui/react-accordion": "^1.2.1",
         "@radix-ui/react-alert-dialog": "^1.1.2",
@@ -52,12 +54,14 @@
         "clsx": "^2.1.1",
         "cmdk": "^1.0.0",
         "date-fns": "^4.1.0",
+        "dotenv": "^16.4.5",
         "embla-carousel-react": "^8.3.0",
         "input-otp": "^1.2.4",
         "lucide-react": "^0.453.0",
         "next": "15.0.1",
         "next-auth": "^5.0.0-beta.25",
         "next-themes": "^0.3.0",
+        "openai": "^4.70.2",
         "react": "19.0.0-rc-69d4b800-20241021",
         "react-day-picker": "^8.10.1",
         "react-dom": "19.0.0-rc-69d4b800-20241021",

src/app/api/embeddings/route.ts

@@ -0,0 +1,45 @@
+import { NextResponse } from "next/server";
+import { index } from "@/lib/pineconeClient"; // adjust the import path as needed
+import { createEmbedding } from "@/lib/openAiClient"; // adjust the import path as needed
+
+export async function POST(request: Request) {
+  const { id, text } = await request.json();
+
+  if (!id || !text) {
+    return NextResponse.json({ error: "Missing id or text" }, { status: 400 });
+  }
+
+  // Generate the embedding
+  const embedding = await createEmbedding(text);
+
+  // Upsert the embedding into Pinecone
+  await index
+    .namespace("ns1")
+    .upsert([{ id, values: embedding, metadata: { text } }]);
+
+  return NextResponse.json({ message: "Embedding uploaded successfully" });
+}
+
+export async function GET(request: Request) {
+  const { searchParams } = new URL(request.url);
+  const queryText = searchParams.get("text");
+
+  if (!queryText) {
+    return NextResponse.json(
+      { error: "Query text is required" },
+      { status: 400 },
+    );
+  }
+
+  // Generate the embedding for the query
+  const queryVector = await createEmbedding(queryText);
+
+  // Search for similar embeddings in Pinecone
+  const response = await index.namespace("ns1").query({
+    topK: 1,
+    vector: queryVector,
+    includeValues: true,
+    includeMetadata: true,
+  });
+  return NextResponse.json(response.matches[0].metadata);
+}

src/lib/openAiClient.ts

@@ -0,0 +1,15 @@
+import { OpenAI } from "openai";
+import { config } from "dotenv";
+config();
+
+const openai = new OpenAI({
+  apiKey: process.env.OPENAI_API_KEY as string,
+});
+
+export const createEmbedding = async (text: string) => {
+  const embedding = await openai.embeddings.create({
+    model: "text-embedding-3-small",
+    input: text,
+  });
+  return embedding.data[0].embedding;
+};

src/lib/pineconeClient.ts

@@ -0,0 +1,9 @@
+import { Pinecone } from "@pinecone-database/pinecone";
+
+const pinecone = new Pinecone({
+  apiKey: process.env.PINECONE_API_KEY as string,
+});
+
+const index = pinecone.index(process.env.PINECONE_INDEX_NAME as string);
+
+export { index };

tsconfig.json

@@ -22,6 +22,12 @@
       "@/*": ["./src/*"]
     }
   },
-  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+  "include": [
+    "next-env.d.ts",
+    "**/*.ts",
+    "**/*.tsx",
+    ".next/types/**/*.ts",
+    "src/lib/openAiClient.js"
+  ],
   "exclude": ["node_modules"]
 }