From b781318c8e26e4af6f6f421ec8020d5130d35654 Mon Sep 17 00:00:00 2001 From: jakubmieszczak Date: Mon, 13 May 2024 22:37:42 +0200 Subject: [PATCH] Add exporting all conversations feature --- api/models/Conversation.js | 9 +++ api/server/routes/convos.js | 49 +++++++++++++- api/server/utils/import/jobDefinition.js | 56 +++++++++++++++- .../components/Nav/SettingsTabs/Data/Data.tsx | 5 +- .../SettingsTabs/Data/ExportConversations.tsx | 66 +++++++++++++++++++ client/src/data-provider/mutations.ts | 61 +++++++++++++++++ client/src/localization/languages/Eng.ts | 5 +- package-lock.json | 9 ++- package.json | 3 + packages/data-provider/src/api-endpoints.ts | 8 +++ packages/data-provider/src/data-service.ts | 14 ++++ packages/data-provider/src/keys.ts | 1 + 12 files changed, 277 insertions(+), 9 deletions(-) create mode 100644 client/src/components/Nav/SettingsTabs/Data/ExportConversations.tsx diff --git a/api/models/Conversation.js b/api/models/Conversation.js index 2dfe8d51cd0..980a5d70803 100644 --- a/api/models/Conversation.js +++ b/api/models/Conversation.js @@ -75,6 +75,15 @@ module.exports = { return { message: 'Error getting conversations' }; } }, + getAllConvos: async (user) => { + try { + const convos = await Conversation.find({ user }).sort({ updatedAt: -1 }).lean(); + return { conversations: convos }; + } catch (error) { + logger.error('[getAllConvos] Error getting conversations', error); + return { message: 'Error getting conversations' }; + } + }, getConvosQueried: async (user, convoIds, pageNumber = 1, pageSize = 25) => { try { if (!convoIds || convoIds.length === 0) { diff --git a/api/server/routes/convos.js b/api/server/routes/convos.js index 275f5a9755e..0eaf4351a6b 100644 --- a/api/server/routes/convos.js +++ b/api/server/routes/convos.js @@ -3,7 +3,10 @@ const express = require('express'); const { CacheKeys } = require('librechat-data-provider'); const { initializeClient } = require('~/server/services/Endpoints/assistants'); const { getConvosByPage, deleteConvos, getConvo, saveConvo } = require('~/models/Conversation'); -const { IMPORT_CONVERSATION_JOB_NAME } = require('~/server/utils/import/jobDefinition'); +const { + IMPORT_CONVERSATION_JOB_NAME, + EXPORT_CONVERSATION_JOB_NAME, +} = require('~/server/utils/import/jobDefinition'); const { storage, importFileFilter } = require('~/server/routes/files/multer'); const requireJwtAuth = require('~/server/middleware/requireJwtAuth'); const { forkConversation } = require('~/server/utils/import/fork'); @@ -12,6 +15,8 @@ const jobScheduler = require('~/server/utils/jobScheduler'); const getLogStores = require('~/cache/getLogStores'); const { sleep } = require('~/server/utils'); const { logger } = require('~/config'); +const os = require('os'); +const path = require('path'); const router = express.Router(); router.use(requireJwtAuth); @@ -168,9 +173,19 @@ router.post('/fork', async (req, res) => { res.status(500).send('Error forking conversation'); } }); +//router.post('/export', importIpLimiter, importUserLimiter, async (req, res) => { +router.post('/export', async (req, res) => { + try { + const job = await jobScheduler.now(EXPORT_CONVERSATION_JOB_NAME, '', req.user.id); + res.status(200).json({ message: 'Export started', jobId: job.id }); + } catch (error) { + console.error('Error exporting conversations', error); + res.status(500).send('Error exporting conversations'); + } +}); -// Get the status of an import job for polling -router.get('/import/jobs/:jobId', async (req, res) => { +// put this in a function +const jobStatusHandler = async (req, res) => { try { const { jobId } = req.params; const { userId, ...jobStatus } = await jobScheduler.getJobStatus(jobId); @@ -187,6 +202,34 @@ router.get('/import/jobs/:jobId', async (req, res) => { logger.error('Error getting job details', error); res.status(500).send('Error getting job details'); } +}; + +// Get the status of an import job for polling +router.get('/import/jobs/:jobId', jobStatusHandler); + +// Get the status of an export job for polling +router.get('/export/jobs/:jobId', jobStatusHandler); + +router.get('/export/jobs/:jobId/conversations.json', async (req, res) => { + logger.info('Downloading JSON file'); + try { + //put this in a function + const { jobId } = req.params; + const tempDir = os.tmpdir(); + const filePath = path.join(tempDir, `export-${jobId}`); + + res.setHeader('Content-Type', 'application/json'); + + res.sendFile(filePath, (err) => { + if (err) { + console.error(err); + res.status(500).send('An error occurred'); + } + }); + } catch (error) { + console.error('Error downloading JSON file', error); + res.status(500).send('Error downloading JSON file'); + } }); module.exports = router; diff --git a/api/server/utils/import/jobDefinition.js b/api/server/utils/import/jobDefinition.js index 7b5d217229f..fa063ad0a8f 100644 --- a/api/server/utils/import/jobDefinition.js +++ b/api/server/utils/import/jobDefinition.js @@ -3,8 +3,13 @@ const jobScheduler = require('~/server/utils/jobScheduler'); const { getImporter } = require('./importers'); const { indexSync } = require('~/lib/db'); const { logger } = require('~/config'); +const { getAllConvos } = require('~/models/Conversation'); +const { getMessages } = require('~/models'); +const os = require('os'); +const path = require('path'); const IMPORT_CONVERSATION_JOB_NAME = 'import conversation'; +const EXPORT_CONVERSATION_JOB_NAME = 'export conversation'; /** * Job definition for importing a conversation. @@ -35,7 +40,56 @@ const importConversationJob = async (job, done) => { } }; +async function createAndDeleteTempFile(content, delay, jobId) { + const tempDir = os.tmpdir(); + const tempFilePath = path.join(tempDir, `export-${jobId}`); + + try { + // Write content to the temporary file using fs.promises API + await fs.writeFile(tempFilePath, JSON.stringify(content)); + console.log(`Temporary file created at: ${tempFilePath}`); + + // Schedule the deletion of the temporary file + setTimeout(async () => { + try { + // Delete the file using fs.promises API + await fs.unlink(tempFilePath); + console.log(`Temporary file deleted: ${tempFilePath}`); + } catch (error) { + console.error('Error deleting the temporary file:', error); + } + }, delay); + + return tempFilePath; + } catch (error) { + console.error('Error handling the temporary file:', error); + } +} + +// Define the export job function +const exportConversationJob = async (job, done) => { + const { requestUserId } = job.attrs.data; + try { + const convos = await getAllConvos(requestUserId); + //const content = req.file.buffer.toString(); + //const job = await jobScheduler.now(EXPORT_CONVERSATION_JOB_NAME, content, req.user.id); + + logger.info('Convos: ' + JSON.stringify(convos)); + + for (let i = 0; i < convos.conversations.length; i++) { + const conversationId = convos.conversations[i].conversationId; + convos.conversations[i].messages = await getMessages({ conversationId }); + } + createAndDeleteTempFile(convos, 5 * 60 * 1000, job.attrs._id); + done(); + } catch (error) { + logger.error('Failed to export conversation: ', error); + done(error); + } +}; + // Call the jobScheduler.define function at startup jobScheduler.define(IMPORT_CONVERSATION_JOB_NAME, importConversationJob); +jobScheduler.define(EXPORT_CONVERSATION_JOB_NAME, exportConversationJob); -module.exports = { IMPORT_CONVERSATION_JOB_NAME }; +module.exports = { IMPORT_CONVERSATION_JOB_NAME, EXPORT_CONVERSATION_JOB_NAME }; diff --git a/client/src/components/Nav/SettingsTabs/Data/Data.tsx b/client/src/components/Nav/SettingsTabs/Data/Data.tsx index 65704d8c126..e90fa743c24 100644 --- a/client/src/components/Nav/SettingsTabs/Data/Data.tsx +++ b/client/src/components/Nav/SettingsTabs/Data/Data.tsx @@ -10,6 +10,7 @@ import { useConversation, useConversations, useOnClickOutside } from '~/hooks'; import ImportConversations from './ImportConversations'; import { ClearChatsButton } from './ClearChats'; import DangerButton from '../DangerButton'; +import ExportConversations from './ExportConversations'; export const RevokeKeysButton = ({ showText = true, @@ -107,10 +108,12 @@ function Data() {
+
+ +
-
([]); + const setError = (error: string) => setErrors((prevErrors) => [...prevErrors, error]); + + const exportFile = useExportConversationsMutation({ + onSuccess: (data) => { + console.log('Exporting started', data); + + showToast({ message: localize('com_ui_export_conversation_success') }); + + // Directly initiate download here if `data` contains downloadable content or URL + downloadConversationsJsonFile(data); + }, + onError: (error) => { + console.error('Error: ', error); + setError( + (error as { response: { data: { message?: string } } }).response.data.message ?? + 'An error occurred while exporting the file.', + ); + showToast({ message: localize('com_ui_export_conversation_error'), status: 'error' }); + }, + }); + + const startExport = () => { + const formdata = new FormData(); + exportFile.mutate(formdata); + }; + + const downloadConversationsJsonFile = (data) => { + // Assuming `data` is the downloadable content; adjust as necessary for your use case + const blob = new Blob([JSON.stringify(data, null, 2)], { type: 'application/json' }); + const url = URL.createObjectURL(blob); + + const link = document.createElement('a'); + link.href = url; + link.download = 'conversations.json'; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + }; + + return ( + <> +
+ {localize('com_nav_export_all_conversations')} + +
+ + ); +} + +export default ExportConversations; diff --git a/client/src/data-provider/mutations.ts b/client/src/data-provider/mutations.ts index ade95fb9022..af28e4d35c9 100644 --- a/client/src/data-provider/mutations.ts +++ b/client/src/data-provider/mutations.ts @@ -272,6 +272,67 @@ export const useUploadConversationsMutation = ( }); }; +export const useExportConversationsMutation = ( + _options?: t.MutationOptions, +) => { + const queryClient = useQueryClient(); + const { onSuccess, onError } = _options || {}; + + const checkJobStatus = async (jobId) => { + try { + const response = await dataService.queryExportAllConversationJobStatus(jobId); + return response; + } catch (error) { + throw new Error('Failed to check job status'); + } + }; + + const pollJobStatus = (jobId, onSuccess, onError) => { + const intervalId = setInterval(async () => { + try { + const statusResponse = await checkJobStatus(jobId); + console.log('Polling job status:', statusResponse); + if (statusResponse.status === 'completed' || statusResponse.status === 'failed') { + clearInterval(intervalId); + if (statusResponse.status === 'completed') { + onSuccess && onSuccess(); + } else { + onError && + onError(new Error(statusResponse.failReason || 'Failed to export conversations')); + } + } + } catch (error) { + clearInterval(intervalId); + onError && onError(error); + } + }, 500); // Poll every 0,5 seconds. Adjust time as necessary. + }; + + return useMutation({ + mutationFn: (formData: FormData) => dataService.exportAllConversationsToJson(formData), + onSuccess: (data) => { + queryClient.invalidateQueries(QueryKeys.allConversations); + const jobId = data.jobId; + if (jobId) { + console.log('Job ID:', jobId); + pollJobStatus( + jobId, + async () => { + queryClient.invalidateQueries(QueryKeys.allConversations); + onSuccess?.(await dataService.exportConversationsFile(jobId)); + }, + (error) => { + onError?.(error, { jobId }, { context: 'ExportJobFailed' }); + }, + ); + } + }, + onError: (err, variables, context) => { + onError?.(err, variables, context); + }, + }); +}; + export const useUploadFileMutation = ( _options?: t.UploadMutationOptions, ): UseMutationResult< diff --git a/client/src/localization/languages/Eng.ts b/client/src/localization/languages/Eng.ts index 315fad50bd1..8f5ca10231d 100644 --- a/client/src/localization/languages/Eng.ts +++ b/client/src/localization/languages/Eng.ts @@ -169,6 +169,8 @@ export default { com_ui_import_conversation_success: 'Conversations imported successfully', com_ui_import_conversation_error: 'There was an error importing your conversations', com_ui_import_conversation_file_type_error: 'Unsupported import type', + com_ui_export_conversation_error: 'There was an error exporting your conversations', + com_ui_export_conversation_success: 'Successfully exported conversations', com_ui_confirm_action: 'Confirm Action', com_ui_chats: 'chats', com_ui_avatar: 'Avatar', @@ -461,7 +463,8 @@ export default { com_nav_export_all_message_branches: 'Export all message branches', com_nav_export_recursive_or_sequential: 'Recursive or sequential?', com_nav_export_recursive: 'Recursive', - com_nav_export_conversation: 'Export conversation', + com_nav_export_conversation: 'Export conversations', + com_nav_export_all_conversations: 'Export all conversations to a JSON file', com_nav_my_files: 'My Files', com_nav_theme: 'Theme', com_nav_theme_system: 'System', diff --git a/package-lock.json b/package-lock.json index eb781bd874f..add3fd783a4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,9 @@ "client", "packages/*" ], + "dependencies": { + "ollama": "^0.5.1" + }, "devDependencies": { "@playwright/test": "^1.38.1", "@typescript-eslint/eslint-plugin": "^5.62.0", @@ -21648,9 +21651,9 @@ } }, "node_modules/ollama": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/ollama/-/ollama-0.5.0.tgz", - "integrity": "sha512-CRtRzsho210EGdK52GrUMohA2pU+7NbgEaBG3DcYeRmvQthDO7E2LHOkLlUUeaYUlNmEd8icbjC02ug9meSYnw==", + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/ollama/-/ollama-0.5.1.tgz", + "integrity": "sha512-mAiCHxdvu63E8EFopz0y82QG7rGfYmKAWgmjG2C7soiRuz/Sj3r/ebvCOp+jasiCubqUPE0ZThKT5LR6wrrPtA==", "dependencies": { "whatwg-fetch": "^3.6.20" } diff --git a/package.json b/package.json index a707d74f3da..188a43d221e 100644 --- a/package.json +++ b/package.json @@ -100,5 +100,8 @@ "admin/", "packages/" ] + }, + "dependencies": { + "ollama": "^0.5.1" } } diff --git a/packages/data-provider/src/api-endpoints.ts b/packages/data-provider/src/api-endpoints.ts index 2e56efb8d0f..13d4eb40883 100644 --- a/packages/data-provider/src/api-endpoints.ts +++ b/packages/data-provider/src/api-endpoints.ts @@ -39,6 +39,14 @@ export const forkConversation = () => `${conversationsRoot}/fork`; export const importConversationJobStatus = (jobId: string) => `${conversationsRoot}/import/jobs/${jobId}`; +export const exportAllConversations = () => `${conversationsRoot}/export`; + +export const exportAllConversationsJobStatus = (jobId: string) => + `${conversationsRoot}/export/jobs/${jobId}`; + +export const downloadExportedConversations = (jobId: string) => + `${conversationsRoot}/export/jobs/${jobId}/conversations.json`; + export const search = (q: string, pageNumber: string) => `/api/search?q=${q}&pageNumber=${pageNumber}`; diff --git a/packages/data-provider/src/data-service.ts b/packages/data-provider/src/data-service.ts index b3346190963..d36af82fe6c 100644 --- a/packages/data-provider/src/data-service.ts +++ b/packages/data-provider/src/data-service.ts @@ -213,6 +213,20 @@ export const queryImportConversationJobStatus = async ( return request.get(endpoints.importConversationJobStatus(jobId)); }; +export const exportAllConversationsToJson = (data: FormData): Promise => { + return request.postMultiPart(endpoints.exportAllConversations(), data); +}; + +export const queryExportAllConversationJobStatus = async ( + jobId: string, +): Promise => { + return request.get(endpoints.exportAllConversationsJobStatus(jobId)); +}; + +export const exportConversationsFile = async (jobId: string): Promise => { + return request.get(endpoints.downloadExportedConversations(jobId)); +}; + export const uploadAvatar = (data: FormData): Promise => { return request.postMultiPart(endpoints.avatar(), data); }; diff --git a/packages/data-provider/src/keys.ts b/packages/data-provider/src/keys.ts index aa664f8c39a..d92aa532837 100644 --- a/packages/data-provider/src/keys.ts +++ b/packages/data-provider/src/keys.ts @@ -2,6 +2,7 @@ export enum QueryKeys { messages = 'messages', allConversations = 'allConversations', archivedConversations = 'archivedConversations', + conversations = 'conversations', //delete this? searchConversations = 'searchConversations', conversation = 'conversation', searchEnabled = 'searchEnabled',