Add exporting all conversations feature

api/models/Conversation.js

@@ -75,6 +75,15 @@ module.exports = {
       return { message: 'Error getting conversations' };
     }
   },
+  getAllConvos: async (user) => {
+    try {
+      const convos = await Conversation.find({ user }).sort({ updatedAt: -1 }).lean();
+      return { conversations: convos };
+    } catch (error) {
+      logger.error('[getAllConvos] Error getting conversations', error);
+      return { message: 'Error getting conversations' };
+    }
+  },
   getConvosQueried: async (user, convoIds, pageNumber = 1, pageSize = 25) => {
     try {
       if (!convoIds || convoIds.length === 0) {

api/server/routes/convos.js

@@ -3,7 +3,10 @@ const express = require('express');
 const { CacheKeys } = require('librechat-data-provider');
 const { initializeClient } = require('~/server/services/Endpoints/assistants');
 const { getConvosByPage, deleteConvos, getConvo, saveConvo } = require('~/models/Conversation');
-const { IMPORT_CONVERSATION_JOB_NAME } = require('~/server/utils/import/jobDefinition');
+const {
+  IMPORT_CONVERSATION_JOB_NAME,
+  EXPORT_CONVERSATION_JOB_NAME,
+} = require('~/server/utils/import/jobDefinition');
 const { storage, importFileFilter } = require('~/server/routes/files/multer');
 const requireJwtAuth = require('~/server/middleware/requireJwtAuth');
 const { forkConversation } = require('~/server/utils/import/fork');
@@ -12,6 +15,8 @@ const jobScheduler = require('~/server/utils/jobScheduler');
 const getLogStores = require('~/cache/getLogStores');
 const { sleep } = require('~/server/utils');
 const { logger } = require('~/config');
+const os = require('os');
+const path = require('path');
 
 const router = express.Router();
 router.use(requireJwtAuth);
@@ -168,9 +173,19 @@ router.post('/fork', async (req, res) => {
     res.status(500).send('Error forking conversation');
   }
 });
+//router.post('/export', importIpLimiter, importUserLimiter, async (req, res) => {
+router.post('/export', async (req, res) => {
+  try {
+    const job = await jobScheduler.now(EXPORT_CONVERSATION_JOB_NAME, '', req.user.id);
+    res.status(200).json({ message: 'Export started', jobId: job.id });
+  } catch (error) {
+    console.error('Error exporting conversations', error);
+    res.status(500).send('Error exporting conversations');
+  }
+});
 
-// Get the status of an import job for polling
-router.get('/import/jobs/:jobId', async (req, res) => {
+// put this in a function
+const jobStatusHandler = async (req, res) => {
   try {
     const { jobId } = req.params;
     const { userId, ...jobStatus } = await jobScheduler.getJobStatus(jobId);
@@ -187,6 +202,34 @@ router.get('/import/jobs/:jobId', async (req, res) => {
     logger.error('Error getting job details', error);
     res.status(500).send('Error getting job details');
   }
+};
+
+// Get the status of an import job for polling
+router.get('/import/jobs/:jobId', jobStatusHandler);
+
+// Get the status of an export job for polling
+router.get('/export/jobs/:jobId', jobStatusHandler);
+
+router.get('/export/jobs/:jobId/conversations.json', async (req, res) => {
+  logger.info('Downloading JSON file');
+  try {
+    //put this in a function
+    const { jobId } = req.params;
+    const tempDir = os.tmpdir();
+    const filePath = path.join(tempDir, `export-${jobId}`);
+
+    res.setHeader('Content-Type', 'application/json');
+
+    res.sendFile(filePath, (err) => {
+      if (err) {
+        console.error(err);
+        res.status(500).send('An error occurred');
+      }
+    });
+  } catch (error) {
+    console.error('Error downloading JSON file', error);
+    res.status(500).send('Error downloading JSON file');
+  }
 });
 
 module.exports = router;

api/server/utils/import/jobDefinition.js

@@ -3,8 +3,13 @@ const jobScheduler = require('~/server/utils/jobScheduler');
 const { getImporter } = require('./importers');
 const { indexSync } = require('~/lib/db');
 const { logger } = require('~/config');
+const { getAllConvos } = require('~/models/Conversation');
+const { getMessages } = require('~/models');
+const os = require('os');
+const path = require('path');
 
 const IMPORT_CONVERSATION_JOB_NAME = 'import conversation';
+const EXPORT_CONVERSATION_JOB_NAME = 'export conversation';
 
 /**
  * Job definition for importing a conversation.
@@ -35,7 +40,56 @@ const importConversationJob = async (job, done) => {
   }
 };
 
+async function createAndDeleteTempFile(content, delay, jobId) {
+  const tempDir = os.tmpdir();
+  const tempFilePath = path.join(tempDir, `export-${jobId}`);
+
+  try {
+    // Write content to the temporary file using fs.promises API
+    await fs.writeFile(tempFilePath, JSON.stringify(content));
+    console.log(`Temporary file created at: ${tempFilePath}`);
+
+    // Schedule the deletion of the temporary file
+    setTimeout(async () => {
+      try {
+        // Delete the file using fs.promises API
+        await fs.unlink(tempFilePath);
+        console.log(`Temporary file deleted: ${tempFilePath}`);
+      } catch (error) {
+        console.error('Error deleting the temporary file:', error);
+      }
+    }, delay);
+
+    return tempFilePath;
+  } catch (error) {
+    console.error('Error handling the temporary file:', error);
+  }
+}
+
+// Define the export job function
+const exportConversationJob = async (job, done) => {
+  const { requestUserId } = job.attrs.data;
+  try {
+    const convos = await getAllConvos(requestUserId);
+    //const content = req.file.buffer.toString();
+    //const job = await jobScheduler.now(EXPORT_CONVERSATION_JOB_NAME, content, req.user.id);
+
+    logger.info('Convos: ' + JSON.stringify(convos));
+
+    for (let i = 0; i < convos.conversations.length; i++) {
+      const conversationId = convos.conversations[i].conversationId;
+      convos.conversations[i].messages = await getMessages({ conversationId });
+    }
+    createAndDeleteTempFile(convos, 5 * 60 * 1000, job.attrs._id);
+    done();
+  } catch (error) {
+    logger.error('Failed to export conversation: ', error);
+    done(error);
+  }
+};
+
 // Call the jobScheduler.define function at startup
 jobScheduler.define(IMPORT_CONVERSATION_JOB_NAME, importConversationJob);
+jobScheduler.define(EXPORT_CONVERSATION_JOB_NAME, exportConversationJob);
 
-module.exports = { IMPORT_CONVERSATION_JOB_NAME };
+module.exports = { IMPORT_CONVERSATION_JOB_NAME, EXPORT_CONVERSATION_JOB_NAME };

client/src/components/Nav/SettingsTabs/Data/Data.tsx

@@ -10,6 +10,7 @@ import { useConversation, useConversations, useOnClickOutside } from '~/hooks';
 import ImportConversations from './ImportConversations';
 import { ClearChatsButton } from './ClearChats';
 import DangerButton from '../DangerButton';
+import ExportConversations from './ExportConversations';
 
 export const RevokeKeysButton = ({
   showText = true,
@@ -107,10 +108,12 @@ function Data() {
         <div className="border-b pb-3 last-of-type:border-b-0 dark:border-gray-600">
           <ImportConversations />
         </div>
+        <div className="border-b pb-3 last-of-type:border-b-0 dark:border-gray-700">
+          <ExportConversations />
+        </div>
         <div className="border-b pb-3 last-of-type:border-b-0 dark:border-gray-600">
           <RevokeKeysButton all={true} />
         </div>
-
         <div className="border-b pb-3 last-of-type:border-b-0 dark:border-gray-600">
           <ClearChatsButton
             confirmClear={confirmClearConvos}

client/src/components/Nav/SettingsTabs/Data/ExportConversations.tsx

@@ -0,0 +1,66 @@
+import React, { useState } from 'react';
+import { ArrowRightFromLine } from 'lucide-react';
+import { useExportConversationsMutation } from '~/data-provider';
+import { useLocalize } from '~/hooks';
+import { useToastContext } from '~/Providers';
+
+function ExportConversations() {
+  const localize = useLocalize();
+  const { showToast } = useToastContext();
+  const [errors, setErrors] = useState<string[]>([]);
+  const setError = (error: string) => setErrors((prevErrors) => [...prevErrors, error]);
+
+  const exportFile = useExportConversationsMutation({
+    onSuccess: (data) => {
+      console.log('Exporting started', data);
+
+      showToast({ message: localize('com_ui_export_conversation_success') });
+
+      // Directly initiate download here if `data` contains downloadable content or URL
+      downloadConversationsJsonFile(data);
+    },
+    onError: (error) => {
+      console.error('Error: ', error);
+      setError(
+        (error as { response: { data: { message?: string } } }).response.data.message ??
+          'An error occurred while exporting the file.',
+      );
+      showToast({ message: localize('com_ui_export_conversation_error'), status: 'error' });
+    },
+  });
+
+  const startExport = () => {
+    const formdata = new FormData();
+    exportFile.mutate(formdata);
+  };
+
+  const downloadConversationsJsonFile = (data) => {
+    // Assuming `data` is the downloadable content; adjust as necessary for your use case
+    const blob = new Blob([JSON.stringify(data, null, 2)], { type: 'application/json' });
+    const url = URL.createObjectURL(blob);
+
+    const link = document.createElement('a');
+    link.href = url;
+    link.download = 'conversations.json';
+    document.body.appendChild(link);
+    link.click();
+    document.body.removeChild(link);
+  };
+
+  return (
+    <>
+      <div className="flex items-center justify-between">
+        <span>{localize('com_nav_export_all_conversations')}</span>
+        <button
+          onClick={startExport}
+          className="flex h-auto cursor-pointer items-center rounded bg-transparent px-2 py-1 text-xs font-medium font-normal transition-colors hover:bg-gray-100 hover:text-green-700 dark:bg-transparent dark:text-white dark:hover:bg-gray-600 dark:hover:text-green-500"
+        >
+          <ArrowRightFromLine className="mr-1 flex w-[22px] items-center stroke-1" />
+          <span>{localize('com_endpoint_export')}</span>
+        </button>
+      </div>
+    </>
+  );
+}
+
+export default ExportConversations;

client/src/data-provider/mutations.ts

@@ -272,6 +272,67 @@ export const useUploadConversationsMutation = (
   });
 };
 
+export const useExportConversationsMutation = (
+  _options?: t.MutationOptions<t.TImportJobStatus, FormData>,
+) => {
+  const queryClient = useQueryClient();
+  const { onSuccess, onError } = _options || {};
+
+  const checkJobStatus = async (jobId) => {
+    try {
+      const response = await dataService.queryExportAllConversationJobStatus(jobId);
+      return response;
+    } catch (error) {
+      throw new Error('Failed to check job status');
+    }
+  };
+
+  const pollJobStatus = (jobId, onSuccess, onError) => {
+    const intervalId = setInterval(async () => {
+      try {
+        const statusResponse = await checkJobStatus(jobId);
+        console.log('Polling job status:', statusResponse);
+        if (statusResponse.status === 'completed' || statusResponse.status === 'failed') {
+          clearInterval(intervalId);
+          if (statusResponse.status === 'completed') {
+            onSuccess && onSuccess();
+          } else {
+            onError &&
+              onError(new Error(statusResponse.failReason || 'Failed to export conversations'));
+          }
+        }
+      } catch (error) {
+        clearInterval(intervalId);
+        onError && onError(error);
+      }
+    }, 500); // Poll every 0,5 seconds. Adjust time as necessary.
+  };
+
+  return useMutation<t.ImportStartResponse, unknown, FormData>({
+    mutationFn: (formData: FormData) => dataService.exportAllConversationsToJson(formData),
+    onSuccess: (data) => {
+      queryClient.invalidateQueries(QueryKeys.allConversations);
+      const jobId = data.jobId;
+      if (jobId) {
+        console.log('Job ID:', jobId);
+        pollJobStatus(
+          jobId,
+          async () => {
+            queryClient.invalidateQueries(QueryKeys.allConversations);
+            onSuccess?.(await dataService.exportConversationsFile(jobId));
+          },
+          (error) => {
+            onError?.(error, { jobId }, { context: 'ExportJobFailed' });
+          },
+        );
+      }
+    },
+    onError: (err, variables, context) => {
+      onError?.(err, variables, context);
+    },
+  });
+};
+
 export const useUploadFileMutation = (
   _options?: t.UploadMutationOptions,
 ): UseMutationResult<

client/src/localization/languages/Eng.ts

@@ -169,6 +169,8 @@ export default {
   com_ui_import_conversation_success: 'Conversations imported successfully',
   com_ui_import_conversation_error: 'There was an error importing your conversations',
   com_ui_import_conversation_file_type_error: 'Unsupported import type',
+  com_ui_export_conversation_error: 'There was an error exporting your conversations',
+  com_ui_export_conversation_success: 'Successfully exported conversations',
   com_ui_confirm_action: 'Confirm Action',
   com_ui_chats: 'chats',
   com_ui_avatar: 'Avatar',
@@ -461,7 +463,8 @@ export default {
   com_nav_export_all_message_branches: 'Export all message branches',
   com_nav_export_recursive_or_sequential: 'Recursive or sequential?',
   com_nav_export_recursive: 'Recursive',
-  com_nav_export_conversation: 'Export conversation',
+  com_nav_export_conversation: 'Export conversations',
+  com_nav_export_all_conversations: 'Export all conversations to a JSON file',
   com_nav_my_files: 'My Files',
   com_nav_theme: 'Theme',
   com_nav_theme_system: 'System',

package.json

@@ -100,5 +100,8 @@
       "admin/",
       "packages/"
     ]
+  },
+  "dependencies": {
+    "ollama": "^0.5.1"
   }
 }

packages/data-provider/src/api-endpoints.ts

@@ -39,6 +39,14 @@ export const forkConversation = () => `${conversationsRoot}/fork`;
 export const importConversationJobStatus = (jobId: string) =>
   `${conversationsRoot}/import/jobs/${jobId}`;
 
+export const exportAllConversations = () => `${conversationsRoot}/export`;
+
+export const exportAllConversationsJobStatus = (jobId: string) =>
+  `${conversationsRoot}/export/jobs/${jobId}`;
+
+export const downloadExportedConversations = (jobId: string) =>
+  `${conversationsRoot}/export/jobs/${jobId}/conversations.json`;
+
 export const search = (q: string, pageNumber: string) =>
   `/api/search?q=${q}&pageNumber=${pageNumber}`;