✨ Add knowledge retrieval by using RAG

config/dependency_decisions.yml

@@ -56,18 +56,20 @@
 - - :approve
   - argparse
   - :who: OSPO @masutaka
-    :why: Python 2.0 license is compatible with Apache-2.0. But License Finder does
+    :why:
+      Python 2.0 license is compatible with Apache-2.0. But License Finder does
       not support the name "Python-2.0". See https://github.com/pivotal/LicenseFinder/pull/1053
     :versions:
-    - 2.0.1
+      - 2.0.1
     :when: 2024-11-28 08:54:56.971593000 Z
 - - :approve
   - jsonify
   - :who: OSPO @masutaka
-    :why: Public Domain is compatible with Apache-2.0. But it is not a software license.
+    :why:
+      Public Domain is compatible with Apache-2.0. But it is not a software license.
       See https://github.com/liam-hq/liam/issues/111
     :versions:
-    - 0.0.1
+      - 0.0.1
     :when: 2024-11-29 03:35:11.884802000 Z
 - - :permit
   - LGPL-3.0-or-later
@@ -84,10 +86,11 @@
 - - :approve
   - spawndamnit
   - :who: OSPO @masutaka
-    :why: Its license is MIT, but it is mis-detected as a "SEE LICENSE IN LICENSE"
+    :why:
+      Its license is MIT, but it is mis-detected as a "SEE LICENSE IN LICENSE"
       license. See https://github.com/jamiebuilds/spawndamnit/pull/11
     :versions:
-    - 3.0.1
+      - 3.0.1
     :when: 2024-11-29 09:06:33.106701000 Z
 - - :permit
   - Mozilla Public License 2.0
@@ -101,3 +104,9 @@
     :why: Compatible with Apache-2.0 license. See https://opensource.org/license/epl-2-0
     :versions: []
     :when: 2024-12-09 09:29:53.604007000 Z
+- - :approve
+  - "@browserbasehq/sdk"
+  - :who: OSPO @masutaka
+    :why: The license is Apache-2.0. License Finder failed to identify the license.
+    :versions: []
+    :when: 2025-03-04 07:55:29.625648000 Z

frontend/apps/migration-web/.env.template

@@ -2,3 +2,5 @@ OPENAI_API_KEY="YOUR_API_KEY"
 LANGFUSE_PUBLIC_KEY=""
 LANGFUSE_SECRET_KEY=""
 LANGFUSE_BASE_URL="https://cloud.langfuse.com"
+SUPABASE_URL="YOUR_SUPABASE_URL"
+SUPABASE_SERVICE_ROLE_KEY="YOUR_SERVICE_ROLE_KEY"

frontend/apps/migration-web/app/api/retrieve/route.ts

@@ -0,0 +1,94 @@
+import { langfuseLangchainHandler, vectorStore } from '@/lib'
+import { PromptTemplate } from '@langchain/core/prompts'
+import { ChatOpenAI } from '@langchain/openai'
+import { HttpResponseOutputParser } from 'langchain/output_parsers'
+import type { NextRequest } from 'next/server'
+
+export const runtime = 'edge'
+
+const RETRIEVAL_TEMPLATE = `You are a knowledgeable assistant that helps users with their questions by providing accurate and helpful information.
+
+I'll provide you with:
+1. The user's query
+2. Relevant context retrieved from our knowledge base
+
+Based on this information, please provide a detailed response that:
+- Directly addresses the user's question
+- Incorporates relevant information from the provided context
+- Provides clear explanations and examples where appropriate
+- Highlights important considerations or best practices
+- Cites specific information from the context when relevant
+
+User Query:
+"""
+{query}
+"""
+
+Relevant Context:
+"""
+{context}
+"""
+
+Instructions for your response:
+1. If the context contains relevant information, use it to enhance your answer
+2. If the context doesn't contain enough information, acknowledge this and provide the best general guidance you can
+3. If you're unsure about something, be transparent about the limitations of your knowledge
+4. Format your response in Markdown to improve readability
+5. Keep your response practical, specific, and actionable
+6. Do not mention that you are using context or reference the retrieval process in your answer
+
+Please provide a comprehensive and helpful response.`
+
+export async function POST(req: NextRequest) {
+  try {
+    const { query } = await req.json()
+
+    if (!query || typeof query !== 'string') {
+      return new Response(
+        JSON.stringify({
+          error: 'Query is not provided or is in an invalid format',
+        }),
+        { status: 400 },
+      )
+    }
+
+    const retriever = vectorStore.asRetriever({
+      searchType: 'similarity',
+      k: 5,
+    })
+
+    const relevantDocs = await retriever.invoke(query)
+    const context = relevantDocs.map((doc) => doc.pageContent).join('\n\n')
+
+    const prompt = PromptTemplate.fromTemplate(RETRIEVAL_TEMPLATE)
+
+    const model = new ChatOpenAI({
+      temperature: 0.7,
+      model: 'gpt-4o-mini',
+    })
+
+    const outputParser = new HttpResponseOutputParser()
+
+    const chain = prompt.pipe(model).pipe(outputParser)
+
+    const stream = await chain.stream(
+      {
+        query: query,
+        context: context,
+      },
+      {
+        callbacks: [langfuseLangchainHandler],
+      },
+    )
+
+    return new Response(stream)
+  } catch (error) {
+    console.error('Error in retrieve API:', error)
+    return new Response(
+      JSON.stringify({
+        error: 'An error occurred while retrieving knowledge',
+      }),
+      { status: 500 },
+    )
+  }
+}

frontend/apps/migration-web/app/api/vectorize/route.ts

@@ -0,0 +1,49 @@
+import { vectorizeText, vectorizeUrl } from '@/lib/vectorization'
+import type { NextRequest } from 'next/server'
+
+export const runtime = 'edge'
+
+export async function POST(req: NextRequest) {
+  try {
+    const { url, text } = await req.json()
+
+    if (url && typeof url === 'string') {
+      const result = await vectorizeUrl(url)
+      return new Response(
+        JSON.stringify({
+          success: true,
+          message: 'URL content vectorized and stored successfully',
+          id: result.documentId,
+          chunkCount: result.chunkCount,
+        }),
+        { status: 200 },
+      )
+    }
+    if (text && typeof text === 'string') {
+      const result = await vectorizeText(text)
+      return new Response(
+        JSON.stringify({
+          success: true,
+          message: 'Text content vectorized and stored successfully',
+          id: result.documentId,
+          chunkCount: result.chunkCount,
+        }),
+        { status: 200 },
+      )
+    }
+    return new Response(
+      JSON.stringify({
+        error: 'Neither URL nor text is provided or is in an invalid format',
+      }),
+      { status: 400 },
+    )
+  } catch (error) {
+    console.error('Error in vectorize API:', error)
+    return new Response(
+      JSON.stringify({
+        error: 'An error occurred while processing the request',
+      }),
+      { status: 500 },
+    )
+  }
+}

frontend/apps/migration-web/app/globals.css

@@ -22,6 +22,17 @@ body {
   font-family: Arial, Helvetica, sans-serif;
   -webkit-font-smoothing: antialiased;
   -moz-osx-font-smoothing: grayscale;
+  display: flex;
+  flex-direction: column;
+  min-height: 100vh;
+}
+
+main {
+  flex: 1;
+  padding: 2rem;
+  width: 100%;
+  max-width: 1200px;
+  margin: 0 auto;
 }
 
 * {

frontend/apps/migration-web/app/knowledge_retrieval/page.module.css

@@ -0,0 +1,15 @@
+.container {
+  display: flex;
+  flex-direction: column;
+  gap: 2rem;
+  padding: 2rem;
+  max-width: 1200px;
+  margin: 0 auto;
+}
+
+.title {
+  font-size: 2rem;
+  font-weight: 600;
+  color: #333;
+  margin-bottom: 1rem;
+}

frontend/apps/migration-web/app/knowledge_retrieval/page.tsx

@@ -0,0 +1,14 @@
+import { KnowledgeRetrievalWindow } from '@/components/KnowledgeRetrievalWindow'
+import styles from './page.module.css'
+
+export default function KnowledgeRetrievalPage() {
+  return (
+    <div className={styles.container}>
+      <h1 className={styles.title}>Database Knowledge Retrieval</h1>
+      <KnowledgeRetrievalWindow
+        endpoint="api/retrieve"
+        placeholder="Describe the database schema change you're planning to make..."
+      />
+    </div>
+  )
+}

frontend/apps/migration-web/app/layout.tsx

@@ -1,6 +1,7 @@
 import type { Metadata } from 'next'
 import type { ReactNode } from 'react'
 import './globals.css'
+import Header from '@/components/Header'
 
 export const metadata: Metadata = {
   title: 'Liam Migration',
@@ -14,7 +15,10 @@ export default function RootLayout({
 }>) {
   return (
     <html lang="en">
-      <body>{children}</body>
+      <body>
+        <Header />
+        <main>{children}</main>
+      </body>
     </html>
   )
 }

frontend/apps/migration-web/app/vectorize/page.module.css

@@ -0,0 +1,14 @@
+.container {
+  display: flex;
+  flex-direction: column;
+  gap: 2rem;
+  padding: 2rem;
+  max-width: 1200px;
+  margin: 0 auto;
+}
+
+.title {
+  font-size: 2rem;
+  font-weight: 600;
+  margin-bottom: 1rem;
+}

frontend/apps/migration-web/app/vectorize/page.tsx

@@ -0,0 +1,13 @@
+import { TextVectorizer } from '@/components/TextVectorizer'
+import { UrlVectorizer } from '@/components/UrlVectorizer'
+import styles from './page.module.css'
+
+export default function VectorizePage() {
+  return (
+    <div className={styles.container}>
+      <h1 className={styles.title}>Vectorize Content</h1>
+      <UrlVectorizer endpoint="api/vectorize" />
+      <TextVectorizer endpoint="api/vectorize" />
+    </div>
+  )
+}

frontend/apps/migration-web/components/Header.module.css

@@ -0,0 +1,66 @@
+.header {
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+  padding: 1rem 2rem;
+  background-color: #f8f9fa;
+  border-bottom: 1px solid #e9ecef;
+  width: 100%;
+}
+
+.logo {
+  display: flex;
+  align-items: center;
+}
+
+.logo h1 {
+  font-size: 1.5rem;
+  font-weight: 600;
+  color: #212529;
+  margin: 0;
+}
+
+.nav {
+  display: flex;
+  gap: 1.5rem;
+}
+
+.navLink {
+  color: #495057;
+  text-decoration: none;
+  font-weight: 500;
+  padding: 0.5rem 0.75rem;
+  border-radius: 4px;
+  transition: all 0.2s ease;
+}
+
+.navLink:hover {
+  color: #212529;
+  background-color: #e9ecef;
+}
+
+.active {
+  color: #0d6efd;
+  background-color: rgba(13, 110, 253, 0.1);
+}
+
+.active:hover {
+  color: #0d6efd;
+  background-color: rgba(13, 110, 253, 0.15);
+}
+
+@media (max-width: 768px) {
+  .header {
+    flex-direction: column;
+    padding: 1rem;
+  }
+
+  .logo {
+    margin-bottom: 1rem;
+  }
+
+  .nav {
+    width: 100%;
+    justify-content: center;
+  }
+}