🔥🚀 feat: CDN (Firebase) & feat: account section (#1438)

* localization + api-endpoint * docs: added firebase documentation * chore: icons * chore: SettingsTabs * feat: account pannel; fix: gear icons * docs: position update * feat: firebase * feat: plugin support * route * fixed bugs with firebase and moved a lot of files * chore(DALLE3): using UUID v4 * feat: support for social strategies; moved '/images' path * fix: data ignored * gitignore update * docs: update firebase guide * refactor: Firebase - use singleton pattern for firebase initialization, initially on server start - reorganize imports, move firebase specific files to own service under Files - rename modules to remove 'avatar' redundancy - fix imports based on changes * ci(DALLE/DALLE3): fix tests to use logger and new expected outputs, add firebase tests * refactor(loadToolWithAuth): pass userId to tool as field * feat(images/parse): feat: Add URL Image Basename Extraction Implement a new module to extract the basename of an image from a given URL. This addition includes the function, which parses the URL and retrieves the basename using the Node.js 'url' and 'path' modules. The function is documented with JSDoc comments for better maintainability and understanding. This feature enhances the application's ability to handle and process image URLs efficiently. * refactor(addImages): function to use a more specific regular expression for observedImagePath based on the generated image markdown standard across the app * refactor(DALLE/DALLE3): utilize `getImageBasename` and `this.userId`; fix: pass correct image path to firebase url helper * fix(addImages): make more general to match any image markdown descriptor * fix(parse/getImageBasename): test result of this function for an actual image basename * ci(DALLE3): mock getImageBasename * refactor(AuthContext): use Recoil atom state for user * feat: useUploadAvatarMutation, react-query hook for avatar upload * fix(Toast): stack z-order of Toast over all components (1000) * refactor(showToast): add optional status field to avoid importing NotificationSeverity on each use of the function * refactor(routes/avatar): remove unnecessary get route, get userId from req.user.id, require auth on POST request * chore(uploadAvatar): TODO: remove direct use of Model, `User` * fix(client): fix Spinner imports * refactor(Avatar): use react-query hook, Toast, remove unnecessary states, add optimistic UI to upload * fix(avatar/localStrategy): correctly save local profile picture and cache bust for immediate rendering; fix: firebase init info message (only show once) * fix: use `includes` instead of `endsWith` for checking manual query of avatar image path in case more queries are appended (as is done in avatar/localStrategy) --------- Co-authored-by: Danny Avila <messagedaniel@protonmail.com>
danny-avila · Dec 30, 2023 · f19f5dc · f19f5dc
1 parent bd4d23d
commit f19f5dc
Show file tree

Hide file tree

Showing 59 changed files with 1,850 additions and 167 deletions.
diff --git a/.env.example b/.env.example
@@ -281,6 +281,17 @@ EMAIL_PASSWORD=
 EMAIL_FROM_NAME=                
 EMAIL_FROM=noreply@librechat.ai
 
+#========================#
+# Firebase CDN           #
+#========================#
+
+FIREBASE_API_KEY=
+FIREBASE_AUTH_DOMAIN=
+FIREBASE_PROJECT_ID=
+FIREBASE_STORAGE_BUCKET=
+FIREBASE_MESSAGING_SENDER_ID=
+FIREBASE_APP_ID=
+
 #==================================================#
 #                      Others                      #
 #==================================================#

diff --git a/.gitignore b/.gitignore
@@ -82,4 +82,6 @@ data.ms/*
 auth.json
 
 /packages/ux-shared/
-/images
+/images
+
+!client/src/components/Nav/SettingsTabs/Data/
diff --git a/api/app/clients/output_parsers/addImages.js b/api/app/clients/output_parsers/addImages.js
@@ -60,12 +60,10 @@ function addImages(intermediateSteps, responseMessage) {
     if (!observation || !observation.includes('![')) {
       return;
     }
-    const observedImagePath = observation.match(/\(\/images\/.*\.\w*\)/g);
+    const observedImagePath = observation.match(/!\[.*\]\([^)]*\)/g);
     if (observedImagePath && !responseMessage.text.includes(observedImagePath[0])) {
       responseMessage.text += '\n' + observation;
-      if (process.env.DEBUG_PLUGINS) {
-        logger.debug('[addImages] added image from intermediateSteps:', observation);
-      }
+      logger.debug('[addImages] added image from intermediateSteps:', observation);
     }
   });
 }

diff --git a/api/app/clients/tools/DALL-E.js b/api/app/clients/tools/DALL-E.js
@@ -4,8 +4,15 @@ const fs = require('fs');
 const path = require('path');
 const OpenAI = require('openai');
 // const { genAzureEndpoint } = require('~/utils/genAzureEndpoints');
+const { v4: uuidv4 } = require('uuid');
 const { Tool } = require('langchain/tools');
 const { HttpsProxyAgent } = require('https-proxy-agent');
+const {
+  saveImageToFirebaseStorage,
+  getFirebaseStorageImageUrl,
+  getFirebaseStorage,
+} = require('~/server/services/Files/Firebase');
+const { getImageBasename } = require('~/server/services/Files/images');
 const extractBaseURL = require('~/utils/extractBaseURL');
 const saveImageFromUrl = require('./saveImageFromUrl');
 const { logger } = require('~/config');
@@ -15,7 +22,9 @@ class OpenAICreateImage extends Tool {
   constructor(fields = {}) {
     super();
 
+    this.userId = fields.userId;
     let apiKey = fields.DALLE_API_KEY || this.getApiKey();
+
     const config = { apiKey };
     if (DALLE_REVERSE_PROXY) {
       config.baseURL = extractBaseURL(DALLE_REVERSE_PROXY);
@@ -24,7 +33,6 @@ class OpenAICreateImage extends Tool {
     if (PROXY) {
       config.httpAgent = new HttpsProxyAgent(PROXY);
     }
-
     // let azureKey = fields.AZURE_API_KEY || process.env.AZURE_API_KEY;
 
     // if (azureKey) {
@@ -97,12 +105,11 @@ Guidelines:
       throw new Error('No image URL returned from OpenAI API.');
     }
 
-    const regex = /img-[\w\d]+.png/;
-    const match = theImageUrl.match(regex);
-    let imageName = '1.png';
+    const imageBasename = getImageBasename(theImageUrl);
+    let imageName = `image_${uuidv4()}.png`;
 
-    if (match) {
-      imageName = match[0];
+    if (imageBasename) {
+      imageName = imageBasename;
       logger.debug('[DALL-E]', { imageName }); // Output: img-lgCf7ppcbhqQrz6a5ear6FOb.png
     } else {
       logger.debug('[DALL-E] No image name found in the string.', {
@@ -111,7 +118,18 @@ Guidelines:
       });
     }
 
-    this.outputPath = path.resolve(__dirname, '..', '..', '..', '..', 'client', 'public', 'images');
+    this.outputPath = path.resolve(
+      __dirname,
+      '..',
+      '..',
+      '..',
+      '..',
+      'client',
+      'public',
+      'images',
+      this.userId,
+    );
+
     const appRoot = path.resolve(__dirname, '..', '..', '..', '..', 'client');
     this.relativeImageUrl = path.relative(appRoot, this.outputPath);
 
@@ -120,14 +138,25 @@ Guidelines:
       fs.mkdirSync(this.outputPath, { recursive: true });
     }
 
-    try {
-      await saveImageFromUrl(theImageUrl, this.outputPath, imageName);
-      this.result = this.getMarkdownImageUrl(imageName);
-    } catch (error) {
-      logger.error('Error while saving the DALL-E image:', error);
-      this.result = theImageUrl;
+    const storage = getFirebaseStorage();
+    if (storage) {
+      try {
+        await saveImageToFirebaseStorage(this.userId, theImageUrl, imageName);
+        this.result = await getFirebaseStorageImageUrl(`${this.userId}/${imageName}`);
+        logger.debug('[DALL-E] result: ' + this.result);
+      } catch (error) {
+        logger.error('Error while saving the image to Firebase Storage:', error);
+        this.result = `Failed to save the image to Firebase Storage. ${error.message}`;
+      }
+    } else {
+      try {
+        await saveImageFromUrl(theImageUrl, this.outputPath, imageName);
+        this.result = this.getMarkdownImageUrl(imageName);
+      } catch (error) {
+        logger.error('Error while saving the image locally:', error);
+        this.result = `Failed to save the image locally. ${error.message}`;
+      }
     }
-
     return this.result;
   }
 }

diff --git a/api/app/clients/tools/structured/DALLE3.js b/api/app/clients/tools/structured/DALLE3.js
@@ -4,17 +4,25 @@ const fs = require('fs');
 const path = require('path');
 const { z } = require('zod');
 const OpenAI = require('openai');
+const { v4: uuidv4 } = require('uuid');
 const { Tool } = require('langchain/tools');
 const { HttpsProxyAgent } = require('https-proxy-agent');
-const saveImageFromUrl = require('../saveImageFromUrl');
+const {
+  saveImageToFirebaseStorage,
+  getFirebaseStorageImageUrl,
+  getFirebaseStorage,
+} = require('~/server/services/Files/Firebase');
+const { getImageBasename } = require('~/server/services/Files/images');
 const extractBaseURL = require('~/utils/extractBaseURL');
+const saveImageFromUrl = require('../saveImageFromUrl');
 const { logger } = require('~/config');
 
 const { DALLE3_SYSTEM_PROMPT, DALLE_REVERSE_PROXY, PROXY } = process.env;
 class DALLE3 extends Tool {
   constructor(fields = {}) {
     super();
 
+    this.userId = fields.userId;
     let apiKey = fields.DALLE_API_KEY || this.getApiKey();
     const config = { apiKey };
     if (DALLE_REVERSE_PROXY) {
@@ -108,12 +116,12 @@ class DALLE3 extends Tool {
         n: 1,
       });
     } catch (error) {
-      return `Something went wrong when trying to generate the image. The DALL-E API may unavailable:
+      return `Something went wrong when trying to generate the image. The DALL-E API may be unavailable:
 Error Message: ${error.message}`;
     }
 
     if (!resp) {
-      return 'Something went wrong when trying to generate the image. The DALL-E API may unavailable';
+      return 'Something went wrong when trying to generate the image. The DALL-E API may be unavailable';
     }
 
     const theImageUrl = resp.data[0].url;
@@ -122,12 +130,11 @@ Error Message: ${error.message}`;
       return 'No image URL returned from OpenAI API. There may be a problem with the API or your configuration.';
     }
 
-    const regex = /img-[\w\d]+.png/;
-    const match = theImageUrl.match(regex);
-    let imageName = '1.png';
+    const imageBasename = getImageBasename(theImageUrl);
+    let imageName = `image_${uuidv4()}.png`;
 
-    if (match) {
-      imageName = match[0];
+    if (imageBasename) {
+      imageName = imageBasename;
       logger.debug('[DALL-E-3]', { imageName }); // Output: img-lgCf7ppcbhqQrz6a5ear6FOb.png
     } else {
       logger.debug('[DALL-E-3] No image name found in the string.', {
@@ -146,6 +153,7 @@ Error Message: ${error.message}`;
       'client',
       'public',
       'images',
+      this.userId,
     );
     const appRoot = path.resolve(__dirname, '..', '..', '..', '..', '..', 'client');
     this.relativeImageUrl = path.relative(appRoot, this.outputPath);
@@ -154,13 +162,24 @@ Error Message: ${error.message}`;
     if (!fs.existsSync(this.outputPath)) {
       fs.mkdirSync(this.outputPath, { recursive: true });
     }
-
-    try {
-      await saveImageFromUrl(theImageUrl, this.outputPath, imageName);
-      this.result = this.getMarkdownImageUrl(imageName);
-    } catch (error) {
-      logger.error('Error while saving the image:', error);
-      this.result = theImageUrl;
+    const storage = getFirebaseStorage();
+    if (storage) {
+      try {
+        await saveImageToFirebaseStorage(this.userId, theImageUrl, imageName);
+        this.result = await getFirebaseStorageImageUrl(`${this.userId}/${imageName}`);
+        logger.debug('[DALL-E-3] result: ' + this.result);
+      } catch (error) {
+        logger.error('Error while saving the image to Firebase Storage:', error);
+        this.result = `Failed to save the image to Firebase Storage. ${error.message}`;
+      }
+    } else {
+      try {
+        await saveImageFromUrl(theImageUrl, this.outputPath, imageName);
+        this.result = this.getMarkdownImageUrl(imageName);
+      } catch (error) {
+        logger.error('Error while saving the image locally:', error);
+        this.result = `Failed to save the image locally. ${error.message}`;
+      }
     }
 
     return this.result;

diff --git a/api/app/clients/tools/structured/specs/DALLE3.spec.js b/api/app/clients/tools/structured/specs/DALLE3.spec.js
@@ -2,11 +2,40 @@ const fs = require('fs');
 const path = require('path');
 const OpenAI = require('openai');
 const DALLE3 = require('../DALLE3');
+const {
+  getFirebaseStorage,
+  saveImageToFirebaseStorage,
+} = require('~/server/services/Files/Firebase');
 const saveImageFromUrl = require('../../saveImageFromUrl');
 const { logger } = require('~/config');
 
 jest.mock('openai');
 
+jest.mock('~/server/services/Files/Firebase', () => ({
+  getFirebaseStorage: jest.fn(),
+  saveImageToFirebaseStorage: jest.fn(),
+  getFirebaseStorageImageUrl: jest.fn(),
+}));
+
+jest.mock('~/server/services/Files/images', () => ({
+  getImageBasename: jest.fn().mockImplementation((url) => {
+    // Split the URL by '/'
+    const parts = url.split('/');
+
+    // Get the last part of the URL
+    const lastPart = parts.pop();
+
+    // Check if the last part of the URL matches the image extension regex
+    const imageExtensionRegex = /\.(jpg|jpeg|png|gif|bmp|tiff|svg)$/i;
+    if (imageExtensionRegex.test(lastPart)) {
+      return lastPart;
+    }
+
+    // If the regex test fails, return an empty string
+    return '';
+  }),
+}));
+
 const generate = jest.fn();
 OpenAI.mockImplementation(() => ({
   images: {
@@ -187,7 +216,48 @@ describe('DALLE3', () => {
     generate.mockResolvedValue(mockResponse);
     saveImageFromUrl.mockRejectedValue(error);
     const result = await dalle._call(mockData);
-    expect(logger.error).toHaveBeenCalledWith('Error while saving the image:', error);
-    expect(result).toBe(mockResponse.data[0].url);
+    expect(logger.error).toHaveBeenCalledWith('Error while saving the image locally:', error);
+    expect(result).toBe('Failed to save the image locally. Error while saving the image');
+  });
+
+  it('should save image to Firebase Storage if Firebase is initialized', async () => {
+    const mockData = {
+      prompt: 'A test prompt',
+    };
+    const mockImageUrl = 'http://example.com/img-test.png';
+    const mockResponse = { data: [{ url: mockImageUrl }] };
+    generate.mockResolvedValue(mockResponse);
+    getFirebaseStorage.mockReturnValue({}); // Simulate Firebase being initialized
+
+    await dalle._call(mockData);
+
+    expect(getFirebaseStorage).toHaveBeenCalled();
+    expect(saveImageToFirebaseStorage).toHaveBeenCalledWith(
+      undefined,
+      mockImageUrl,
+      expect.any(String),
+    );
+  });
+
+  it('should handle error when saving image to Firebase Storage fails', async () => {
+    const mockData = {
+      prompt: 'A test prompt',
+    };
+    const mockImageUrl = 'http://example.com/img-test.png';
+    const mockResponse = { data: [{ url: mockImageUrl }] };
+    const error = new Error('Error while saving to Firebase');
+    generate.mockResolvedValue(mockResponse);
+    getFirebaseStorage.mockReturnValue({}); // Simulate Firebase being initialized
+    saveImageToFirebaseStorage.mockRejectedValue(error);
+
+    const result = await dalle._call(mockData);
+
+    expect(logger.error).toHaveBeenCalledWith(
+      'Error while saving the image to Firebase Storage:',
+      error,
+    );
+    expect(result).toBe(
+      'Failed to save the image to Firebase Storage. Error while saving to Firebase',
+    );
   });
 });
diff --git a/api/app/clients/tools/util/handleTools.js b/api/app/clients/tools/util/handleTools.js
@@ -67,19 +67,19 @@ const validateTools = async (user, tools = []) => {
   }
 };
 
-const loadToolWithAuth = async (user, authFields, ToolConstructor, options = {}) => {
+const loadToolWithAuth = async (userId, authFields, ToolConstructor, options = {}) => {
   return async function () {
     let authValues = {};
 
     for (const authField of authFields) {
       let authValue = process.env[authField];
       if (!authValue) {
-        authValue = await getUserPluginAuthValue(user, authField);
+        authValue = await getUserPluginAuthValue(userId, authField);
       }
       authValues[authField] = authValue;
     }
 
-    return new ToolConstructor({ ...options, ...authValues });
+    return new ToolConstructor({ ...options, ...authValues, userId });
   };
 };
 

diff --git a/api/package.json b/api/package.json
@@ -44,6 +44,7 @@
     "express-mongo-sanitize": "^2.2.0",
     "express-rate-limit": "^6.9.0",
     "express-session": "^1.17.3",
+    "firebase": "^10.6.0",
     "googleapis": "^126.0.1",
     "handlebars": "^4.7.7",
     "html": "^1.0.0",

diff --git a/api/server/index.js b/api/server/index.js
@@ -4,6 +4,7 @@ const cors = require('cors');
 const express = require('express');
 const passport = require('passport');
 const mongoSanitize = require('express-mongo-sanitize');
+const { initializeFirebase } = require('~/server/services/Files/Firebase/initialize');
 const errorController = require('./controllers/ErrorController');
 const configureSocialLogins = require('./socialLogins');
 const { connectDb, indexSync } = require('~/lib/db');
@@ -23,6 +24,7 @@ const { jwtLogin, passportLogin } = require('~/strategies');
 const startServer = async () => {
   await connectDb();
   logger.info('Connected to MongoDB');
+  initializeFirebase();
   await indexSync();
 
   const app = express();