From f37124f241c9fd2cf310d1bae469bd92e3d99026 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 26 Feb 2025 09:39:38 -0500 Subject: [PATCH 01/18] chore: include all assets for service worker, remove unused tsconfig.node.json, eslint ignore vite config --- client/tsconfig.json | 5 ----- client/tsconfig.node.json | 9 --------- client/vite.config.ts | 20 ++++++-------------- eslint.config.mjs | 1 + 4 files changed, 7 insertions(+), 28 deletions(-) delete mode 100644 client/tsconfig.node.json diff --git a/client/tsconfig.json b/client/tsconfig.json index d588c5afea0..ba00b0d24c4 100644 --- a/client/tsconfig.json +++ b/client/tsconfig.json @@ -35,10 +35,5 @@ "test/setupTests.js", "env.d.ts", "../config/translations/**/*.ts" - ], - "references": [ - { - "path": "./tsconfig.node.json" - } ] } diff --git a/client/tsconfig.node.json b/client/tsconfig.node.json deleted file mode 100644 index 9d31e2aed93..00000000000 --- a/client/tsconfig.node.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "compilerOptions": { - "composite": true, - "module": "ESNext", - "moduleResolution": "Node", - "allowSyntheticDefaultImports": true - }, - "include": ["vite.config.ts"] -} diff --git a/client/vite.config.ts b/client/vite.config.ts index 94c68bfd544..c2a2ecb4674 100644 --- a/client/vite.config.ts +++ b/client/vite.config.ts @@ -37,13 +37,11 @@ export default defineConfig({ }, useCredentials: true, workbox: { - globPatterns: [ - 'assets/**/*.{png,jpg,svg,ico}', - '**/*.{js,css,html,ico,woff2}', - ], + globPatterns: ['**/*'], maximumFileSizeToCacheInBytes: 4 * 1024 * 1024, navigateFallbackDenylist: [/^\/oauth/], }, + includeAssets: ['**/*'], manifest: { name: 'LibreChat', short_name: 'LibreChat', @@ -70,7 +68,7 @@ export default defineConfig({ { src: '/assets/icon-192x192.png', sizes: '192x192', - type: 'image/png' + type: 'image/png', }, { src: '/assets/maskable-icon.png', @@ -113,10 +111,7 @@ export default defineConfig({ if (id.includes('node_modules/highlight.js')) { return 'markdown_highlight'; } - if ( - id.includes('node_modules/hast-util-raw') || - id.includes('node_modules/katex') - ) { + if (id.includes('node_modules/hast-util-raw') || id.includes('node_modules/katex')) { return 'markdown_large'; } // Group TanStack libraries together. @@ -141,10 +136,7 @@ export default defineConfig({ entryFileNames: 'assets/[name].[hash].js', chunkFileNames: 'assets/[name].[hash].js', assetFileNames: (assetInfo) => { - if ( - assetInfo.names && - /\.(woff|woff2|eot|ttf|otf)$/.test(assetInfo.names) - ) { + if (assetInfo.names && /\.(woff|woff2|eot|ttf|otf)$/.test(assetInfo.names)) { return 'assets/fonts/[name][extname]'; } return 'assets/[name].[hash][extname]'; @@ -187,4 +179,4 @@ export function sourcemapExclude(opts?: SourcemapExclude): Plugin { } }, }; -} \ No newline at end of file +} diff --git a/eslint.config.mjs b/eslint.config.mjs index 4ebe862c6f7..dc2567e9b99 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -25,6 +25,7 @@ const compat = new FlatCompat({ export default [ { ignores: [ + 'client/vite.config.ts', 'client/dist/**/*', 'client/public/**/*', 'client/coverage/**/*', From c1174eb7279cb836516097fced92c13c943c03e1 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 26 Feb 2025 11:27:30 -0500 Subject: [PATCH 02/18] chore: exclude image files from service worker caching --- client/vite.config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/client/vite.config.ts b/client/vite.config.ts index c2a2ecb4674..adac3e70e83 100644 --- a/client/vite.config.ts +++ b/client/vite.config.ts @@ -38,6 +38,7 @@ export default defineConfig({ useCredentials: true, workbox: { globPatterns: ['**/*'], + globIgnores: ['images/**/*'], maximumFileSizeToCacheInBytes: 4 * 1024 * 1024, navigateFallbackDenylist: [/^\/oauth/], }, From 4319301ac28cb9681fb0788918326e060c820a29 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 26 Feb 2025 11:36:38 -0500 Subject: [PATCH 03/18] refactor: simplify googleSchema transformation and error handling --- packages/data-provider/src/schemas.ts | 33 ++------------------------- 1 file changed, 2 insertions(+), 31 deletions(-) diff --git a/packages/data-provider/src/schemas.ts b/packages/data-provider/src/schemas.ts index f99bcf1c046..9dd7ee2dabb 100644 --- a/packages/data-provider/src/schemas.ts +++ b/packages/data-provider/src/schemas.ts @@ -760,37 +760,8 @@ export const googleSchema = tConversationSchema spec: true, maxContextTokens: true, }) - .transform((obj) => { - return { - ...obj, - model: obj.model ?? google.model.default, - modelLabel: obj.modelLabel ?? null, - promptPrefix: obj.promptPrefix ?? null, - examples: obj.examples ?? [{ input: { content: '' }, output: { content: '' } }], - temperature: obj.temperature ?? google.temperature.default, - maxOutputTokens: obj.maxOutputTokens ?? google.maxOutputTokens.default, - topP: obj.topP ?? google.topP.default, - topK: obj.topK ?? google.topK.default, - iconURL: obj.iconURL ?? undefined, - greeting: obj.greeting ?? undefined, - spec: obj.spec ?? undefined, - maxContextTokens: obj.maxContextTokens ?? undefined, - }; - }) - .catch(() => ({ - model: google.model.default, - modelLabel: null, - promptPrefix: null, - examples: [{ input: { content: '' }, output: { content: '' } }], - temperature: google.temperature.default, - maxOutputTokens: google.maxOutputTokens.default, - topP: google.topP.default, - topK: google.topK.default, - iconURL: undefined, - greeting: undefined, - spec: undefined, - maxContextTokens: undefined, - })); + .transform((obj: Partial) => removeNullishValues(obj)) + .catch(() => ({})); /** * TODO: Map the following fields: From 4eb80c4acc53f0a60c3af7fe63eb518aec4bd81a Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 26 Feb 2025 11:49:37 -0500 Subject: [PATCH 04/18] fix: max output tokens cap for 3.7 models --- api/app/clients/AnthropicClient.js | 4 +++- packages/data-provider/src/schemas.ts | 14 +++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/api/app/clients/AnthropicClient.js b/api/app/clients/AnthropicClient.js index 1f7eb90b26d..2619c6c23e6 100644 --- a/api/app/clients/AnthropicClient.js +++ b/api/app/clients/AnthropicClient.js @@ -105,7 +105,9 @@ class AnthropicClient extends BaseClient { const modelMatch = matchModelName(this.modelOptions.model, EModelEndpoint.anthropic); this.isClaude3 = modelMatch.includes('claude-3'); - this.isLegacyOutput = !modelMatch.includes('claude-3-5-sonnet'); + this.isLegacyOutput = !( + /claude-3[-.]5-sonnet/.test(modelMatch) || /claude-3[-.]7/.test(modelMatch) + ); this.supportsCacheControl = this.options.promptCache && checkPromptCacheSupport(modelMatch); if ( diff --git a/packages/data-provider/src/schemas.ts b/packages/data-provider/src/schemas.ts index 9dd7ee2dabb..cee0230386c 100644 --- a/packages/data-provider/src/schemas.ts +++ b/packages/data-provider/src/schemas.ts @@ -252,7 +252,8 @@ export const googleSettings = { }, }; -const ANTHROPIC_MAX_OUTPUT = 8192; +const ANTHROPIC_MAX_OUTPUT = 128000; +const DEFAULT_MAX_OUTPUT = 8192; const LEGACY_ANTHROPIC_MAX_OUTPUT = 4096; export const anthropicSettings = { model: { @@ -280,16 +281,19 @@ export const anthropicSettings = { min: 1, max: ANTHROPIC_MAX_OUTPUT, step: 1, - default: ANTHROPIC_MAX_OUTPUT, + default: DEFAULT_MAX_OUTPUT, reset: (modelName: string) => { - if (modelName.includes('claude-3-5-sonnet') || modelName.includes('claude-3-7-sonnet')) { - return ANTHROPIC_MAX_OUTPUT; + if (/claude-3[-.]5-sonnet/.test(modelName) || /claude-3[-.]7/.test(modelName)) { + return DEFAULT_MAX_OUTPUT; } return 4096; }, set: (value: number, modelName: string) => { - if (!modelName.includes('claude-3-5-sonnet') && value > LEGACY_ANTHROPIC_MAX_OUTPUT) { + if ( + !(/claude-3[-.]5-sonnet/.test(modelName) || /claude-3[-.]7/.test(modelName)) && + value > LEGACY_ANTHROPIC_MAX_OUTPUT + ) { return LEGACY_ANTHROPIC_MAX_OUTPUT; } From 67586a1a06bab8ecf14488709265e56dae865a7b Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 26 Feb 2025 11:55:13 -0500 Subject: [PATCH 05/18] fix: skip index fixing in CI, development, and test environments --- api/models/Token.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/api/models/Token.js b/api/models/Token.js index 210666ddd78..0ed18320aee 100644 --- a/api/models/Token.js +++ b/api/models/Token.js @@ -13,6 +13,13 @@ const Token = mongoose.model('Token', tokenSchema); */ async function fixIndexes() { try { + if ( + process.env.NODE_ENV === 'CI' || + process.env.NODE_ENV === 'development' || + process.env.NODE_ENV === 'test' + ) { + return; + } const indexes = await Token.collection.indexes(); logger.debug('Existing Token Indexes:', JSON.stringify(indexes, null, 2)); const unwantedTTLIndexes = indexes.filter( From bcd5cf854e60137b6c3792695d05419fe6567ef7 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 26 Feb 2025 11:55:38 -0500 Subject: [PATCH 06/18] ci: add maxOutputTokens handling tests for Claude models --- api/app/clients/specs/AnthropicClient.test.js | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/api/app/clients/specs/AnthropicClient.test.js b/api/app/clients/specs/AnthropicClient.test.js index eef6bb6748a..0d4a9535657 100644 --- a/api/app/clients/specs/AnthropicClient.test.js +++ b/api/app/clients/specs/AnthropicClient.test.js @@ -405,4 +405,111 @@ describe('AnthropicClient', () => { expect(Number.isNaN(result)).toBe(false); }); }); + + describe('maxOutputTokens handling for different models', () => { + it('should not cap maxOutputTokens for Claude 3.5 Sonnet models', () => { + const client = new AnthropicClient('test-api-key'); + const highTokenValue = anthropicSettings.legacy.maxOutputTokens.default * 10; + + client.setOptions({ + modelOptions: { + model: 'claude-3-5-sonnet', + maxOutputTokens: highTokenValue, + }, + }); + + expect(client.modelOptions.maxOutputTokens).toBe(highTokenValue); + + // Test with decimal notation + client.setOptions({ + modelOptions: { + model: 'claude-3.5-sonnet', + maxOutputTokens: highTokenValue, + }, + }); + + expect(client.modelOptions.maxOutputTokens).toBe(highTokenValue); + }); + + it('should not cap maxOutputTokens for Claude 3.7 models', () => { + const client = new AnthropicClient('test-api-key'); + const highTokenValue = anthropicSettings.legacy.maxOutputTokens.default * 2; + + client.setOptions({ + modelOptions: { + model: 'claude-3-7-sonnet', + maxOutputTokens: highTokenValue, + }, + }); + + expect(client.modelOptions.maxOutputTokens).toBe(highTokenValue); + + // Test with decimal notation + client.setOptions({ + modelOptions: { + model: 'claude-3.7-sonnet', + maxOutputTokens: highTokenValue, + }, + }); + + expect(client.modelOptions.maxOutputTokens).toBe(highTokenValue); + }); + + it('should cap maxOutputTokens for Claude 3.5 Haiku models', () => { + const client = new AnthropicClient('test-api-key'); + const highTokenValue = anthropicSettings.legacy.maxOutputTokens.default * 2; + + client.setOptions({ + modelOptions: { + model: 'claude-3-5-haiku', + maxOutputTokens: highTokenValue, + }, + }); + + expect(client.modelOptions.maxOutputTokens).toBe( + anthropicSettings.legacy.maxOutputTokens.default, + ); + + // Test with decimal notation + client.setOptions({ + modelOptions: { + model: 'claude-3.5-haiku', + maxOutputTokens: highTokenValue, + }, + }); + + expect(client.modelOptions.maxOutputTokens).toBe( + anthropicSettings.legacy.maxOutputTokens.default, + ); + }); + + it('should cap maxOutputTokens for Claude 3 Haiku and Opus models', () => { + const client = new AnthropicClient('test-api-key'); + const highTokenValue = anthropicSettings.legacy.maxOutputTokens.default * 2; + + // Test haiku + client.setOptions({ + modelOptions: { + model: 'claude-3-haiku', + maxOutputTokens: highTokenValue, + }, + }); + + expect(client.modelOptions.maxOutputTokens).toBe( + anthropicSettings.legacy.maxOutputTokens.default, + ); + + // Test opus + client.setOptions({ + modelOptions: { + model: 'claude-3-opus', + maxOutputTokens: highTokenValue, + }, + }); + + expect(client.modelOptions.maxOutputTokens).toBe( + anthropicSettings.legacy.maxOutputTokens.default, + ); + }); + }); }); From d3c3e1d0f2f4a2719d3c54a2d62331aafa694f2b Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 26 Feb 2025 12:05:58 -0500 Subject: [PATCH 07/18] refactor: drop top_k and top_p parameters for claude-3.7 in AnthropicClient and add tests for new behavior --- api/app/clients/AnthropicClient.js | 11 +- api/app/clients/specs/AnthropicClient.test.js | 168 ++++++++++++++++++ 2 files changed, 177 insertions(+), 2 deletions(-) diff --git a/api/app/clients/AnthropicClient.js b/api/app/clients/AnthropicClient.js index 2619c6c23e6..492def135e7 100644 --- a/api/app/clients/AnthropicClient.js +++ b/api/app/clients/AnthropicClient.js @@ -735,10 +735,17 @@ class AnthropicClient extends BaseClient { stop_sequences, temperature, metadata, - top_p, - top_k, }; + if (!/claude-3[-.]7/.test(model)) { + if (top_p !== undefined) { + requestOptions.top_p = top_p; + } + if (top_k !== undefined) { + requestOptions.top_k = top_k; + } + } + if (this.useMessages) { requestOptions.messages = payload; requestOptions.max_tokens = diff --git a/api/app/clients/specs/AnthropicClient.test.js b/api/app/clients/specs/AnthropicClient.test.js index 0d4a9535657..b565e6d1882 100644 --- a/api/app/clients/specs/AnthropicClient.test.js +++ b/api/app/clients/specs/AnthropicClient.test.js @@ -1,3 +1,4 @@ +const { SplitStreamHandler } = require('@librechat/agents'); const { anthropicSettings } = require('librechat-data-provider'); const AnthropicClient = require('~/app/clients/AnthropicClient'); @@ -512,4 +513,171 @@ describe('AnthropicClient', () => { ); }); }); + + describe('topK/topP parameters for different models', () => { + beforeEach(() => { + // Mock the SplitStreamHandler + jest.spyOn(SplitStreamHandler.prototype, 'handle').mockImplementation(() => {}); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('should include top_k and top_p parameters for non-claude-3.7 models', async () => { + const client = new AnthropicClient('test-api-key'); + + // Create a mock async generator function + async function* mockAsyncGenerator() { + yield { type: 'message_start', message: { usage: {} } }; + yield { delta: { text: 'Test response' } }; + yield { type: 'message_delta', usage: {} }; + } + + // Mock createResponse to return the async generator + jest.spyOn(client, 'createResponse').mockImplementation(() => { + return mockAsyncGenerator(); + }); + + client.setOptions({ + modelOptions: { + model: 'claude-3-opus', + temperature: 0.7, + topK: 10, + topP: 0.9, + }, + }); + + // Mock getClient to capture the request options + let capturedOptions = null; + jest.spyOn(client, 'getClient').mockImplementation((options) => { + capturedOptions = options; + return {}; + }); + + const payload = [{ role: 'user', content: 'Test message' }]; + await client.sendCompletion(payload, {}); + + // Check the options passed to getClient + expect(capturedOptions).toHaveProperty('top_k', 10); + expect(capturedOptions).toHaveProperty('top_p', 0.9); + }); + + it('should include top_k and top_p parameters for claude-3-5-sonnet models', async () => { + const client = new AnthropicClient('test-api-key'); + + // Create a mock async generator function + async function* mockAsyncGenerator() { + yield { type: 'message_start', message: { usage: {} } }; + yield { delta: { text: 'Test response' } }; + yield { type: 'message_delta', usage: {} }; + } + + // Mock createResponse to return the async generator + jest.spyOn(client, 'createResponse').mockImplementation(() => { + return mockAsyncGenerator(); + }); + + client.setOptions({ + modelOptions: { + model: 'claude-3-5-sonnet', + temperature: 0.7, + topK: 10, + topP: 0.9, + }, + }); + + // Mock getClient to capture the request options + let capturedOptions = null; + jest.spyOn(client, 'getClient').mockImplementation((options) => { + capturedOptions = options; + return {}; + }); + + const payload = [{ role: 'user', content: 'Test message' }]; + await client.sendCompletion(payload, {}); + + // Check the options passed to getClient + expect(capturedOptions).toHaveProperty('top_k', 10); + expect(capturedOptions).toHaveProperty('top_p', 0.9); + }); + + it('should not include top_k and top_p parameters for claude-3-7-sonnet models', async () => { + const client = new AnthropicClient('test-api-key'); + + // Create a mock async generator function + async function* mockAsyncGenerator() { + yield { type: 'message_start', message: { usage: {} } }; + yield { delta: { text: 'Test response' } }; + yield { type: 'message_delta', usage: {} }; + } + + // Mock createResponse to return the async generator + jest.spyOn(client, 'createResponse').mockImplementation(() => { + return mockAsyncGenerator(); + }); + + client.setOptions({ + modelOptions: { + model: 'claude-3-7-sonnet', + temperature: 0.7, + topK: 10, + topP: 0.9, + }, + }); + + // Mock getClient to capture the request options + let capturedOptions = null; + jest.spyOn(client, 'getClient').mockImplementation((options) => { + capturedOptions = options; + return {}; + }); + + const payload = [{ role: 'user', content: 'Test message' }]; + await client.sendCompletion(payload, {}); + + // Check the options passed to getClient + expect(capturedOptions).not.toHaveProperty('top_k'); + expect(capturedOptions).not.toHaveProperty('top_p'); + }); + + it('should not include top_k and top_p parameters for models with decimal notation (claude-3.7)', async () => { + const client = new AnthropicClient('test-api-key'); + + // Create a mock async generator function + async function* mockAsyncGenerator() { + yield { type: 'message_start', message: { usage: {} } }; + yield { delta: { text: 'Test response' } }; + yield { type: 'message_delta', usage: {} }; + } + + // Mock createResponse to return the async generator + jest.spyOn(client, 'createResponse').mockImplementation(() => { + return mockAsyncGenerator(); + }); + + client.setOptions({ + modelOptions: { + model: 'claude-3.7-sonnet', + temperature: 0.7, + topK: 10, + topP: 0.9, + }, + }); + + // Mock getClient to capture the request options + let capturedOptions = null; + jest.spyOn(client, 'getClient').mockImplementation((options) => { + capturedOptions = options; + return {}; + }); + + const payload = [{ role: 'user', content: 'Test message' }]; + await client.sendCompletion(payload, {}); + + // Check the options passed to getClient + expect(capturedOptions).not.toHaveProperty('top_k'); + expect(capturedOptions).not.toHaveProperty('top_p'); + }); + }); }); From 1d1bccd0ca48be9fc82cceff5c2ac9a3baa7ce9c Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 26 Feb 2025 12:09:37 -0500 Subject: [PATCH 08/18] refactor: conditionally include top_k and top_p parameters for non-claude-3.7 models --- api/server/services/Endpoints/anthropic/llm.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/api/server/services/Endpoints/anthropic/llm.js b/api/server/services/Endpoints/anthropic/llm.js index c68fd4738d8..186444cec8c 100644 --- a/api/server/services/Endpoints/anthropic/llm.js +++ b/api/server/services/Endpoints/anthropic/llm.js @@ -43,14 +43,21 @@ function getLLMConfig(apiKey, options = {}) { model: mergedOptions.model, stream: mergedOptions.stream, temperature: mergedOptions.temperature, - topP: mergedOptions.topP, - topK: mergedOptions.topK, stopSequences: mergedOptions.stop, maxTokens: mergedOptions.maxOutputTokens || anthropicSettings.maxOutputTokens.reset(mergedOptions.model), clientOptions: {}, }; + if (!/claude-3[-.]7/.test(mergedOptions.model)) { + if (mergedOptions.topP !== undefined) { + requestOptions.topP = mergedOptions.topP; + } + if (mergedOptions.topK !== undefined) { + requestOptions.topK = mergedOptions.topK; + } + } + const supportsCacheControl = systemOptions.promptCache === true && checkPromptCacheSupport(requestOptions.model); const headers = getClaudeHeaders(requestOptions.model, supportsCacheControl); From 0f2139dded22dc98066e2305857fcf15a88ce0a1 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 26 Feb 2025 12:13:33 -0500 Subject: [PATCH 09/18] ci: add unit tests for getLLMConfig function with various model options --- .../services/Endpoints/anthropic/llm.spec.js | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 api/server/services/Endpoints/anthropic/llm.spec.js diff --git a/api/server/services/Endpoints/anthropic/llm.spec.js b/api/server/services/Endpoints/anthropic/llm.spec.js new file mode 100644 index 00000000000..a1dc6a44b6a --- /dev/null +++ b/api/server/services/Endpoints/anthropic/llm.spec.js @@ -0,0 +1,112 @@ +const { anthropicSettings } = require('librechat-data-provider'); +const { getLLMConfig } = require('~/server/services/Endpoints/anthropic/llm'); + +jest.mock('https-proxy-agent', () => ({ + HttpsProxyAgent: jest.fn().mockImplementation((proxy) => ({ proxy })), +})); + +describe('getLLMConfig', () => { + it('should create a basic configuration with default values', () => { + const result = getLLMConfig('test-api-key', { modelOptions: {} }); + + expect(result.llmConfig).toHaveProperty('apiKey', 'test-api-key'); + expect(result.llmConfig).toHaveProperty('model', anthropicSettings.model.default); + expect(result.llmConfig).toHaveProperty('stream', true); + expect(result.llmConfig).toHaveProperty('maxTokens'); + }); + + it('should include proxy settings when provided', () => { + const result = getLLMConfig('test-api-key', { + modelOptions: {}, + proxy: 'http://proxy:8080', + }); + + expect(result.llmConfig.clientOptions).toHaveProperty('httpAgent'); + expect(result.llmConfig.clientOptions.httpAgent).toHaveProperty('proxy', 'http://proxy:8080'); + }); + + it('should include reverse proxy URL when provided', () => { + const result = getLLMConfig('test-api-key', { + modelOptions: {}, + reverseProxyUrl: 'http://reverse-proxy', + }); + + expect(result.llmConfig.clientOptions).toHaveProperty('baseURL', 'http://reverse-proxy'); + }); + + it('should include topK and topP for non-Claude-3.7 models', () => { + const result = getLLMConfig('test-api-key', { + modelOptions: { + model: 'claude-3-opus', + topK: 10, + topP: 0.9, + }, + }); + + expect(result.llmConfig).toHaveProperty('topK', 10); + expect(result.llmConfig).toHaveProperty('topP', 0.9); + }); + + it('should include topK and topP for Claude-3.5 models', () => { + const result = getLLMConfig('test-api-key', { + modelOptions: { + model: 'claude-3-5-sonnet', + topK: 10, + topP: 0.9, + }, + }); + + expect(result.llmConfig).toHaveProperty('topK', 10); + expect(result.llmConfig).toHaveProperty('topP', 0.9); + }); + + it('should NOT include topK and topP for Claude-3-7 models (hyphen notation)', () => { + const result = getLLMConfig('test-api-key', { + modelOptions: { + model: 'claude-3-7-sonnet', + topK: 10, + topP: 0.9, + }, + }); + + expect(result.llmConfig).not.toHaveProperty('topK'); + expect(result.llmConfig).not.toHaveProperty('topP'); + }); + + it('should NOT include topK and topP for Claude-3.7 models (decimal notation)', () => { + const result = getLLMConfig('test-api-key', { + modelOptions: { + model: 'claude-3.7-sonnet', + topK: 10, + topP: 0.9, + }, + }); + + expect(result.llmConfig).not.toHaveProperty('topK'); + expect(result.llmConfig).not.toHaveProperty('topP'); + }); + + it('should handle custom maxOutputTokens', () => { + const result = getLLMConfig('test-api-key', { + modelOptions: { + model: 'claude-3-opus', + maxOutputTokens: 2048, + }, + }); + + expect(result.llmConfig).toHaveProperty('maxTokens', 2048); + }); + + it('should handle promptCache setting', () => { + const result = getLLMConfig('test-api-key', { + modelOptions: { + model: 'claude-3-5-sonnet', + promptCache: true, + }, + }); + + // We're not checking specific header values since that depends on the actual helper function + // Just verifying that the promptCache setting is processed + expect(result.llmConfig).toBeDefined(); + }); +}); From 76365b53a5d6003e8dcf23ba5bc1ff4de6a6ad06 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 26 Feb 2025 12:15:54 -0500 Subject: [PATCH 10/18] chore: remove all OPENROUTER_API_KEY legacy logic --- .env.example | 6 ------ api/app/clients/OpenAIClient.js | 7 +------ api/app/clients/specs/OpenAIClient.test.js | 9 --------- api/server/services/ModelService.js | 5 +---- api/server/services/ModelService.spec.js | 16 ---------------- librechat.example.yaml | 1 - 6 files changed, 2 insertions(+), 42 deletions(-) diff --git a/.env.example b/.env.example index 5fb73557db8..94a6d80d888 100644 --- a/.env.example +++ b/.env.example @@ -209,12 +209,6 @@ ASSISTANTS_API_KEY=user_provided # More info, including how to enable use of Assistants with Azure here: # https://www.librechat.ai/docs/configuration/librechat_yaml/ai_endpoints/azure#using-assistants-with-azure -#============# -# OpenRouter # -#============# -# !!!Warning: Use the variable above instead of this one. Using this one will override the OpenAI endpoint -# OPENROUTER_API_KEY= - #============# # Plugins # #============# diff --git a/api/app/clients/OpenAIClient.js b/api/app/clients/OpenAIClient.js index 7bd7879dcfb..8d0bce25d2a 100644 --- a/api/app/clients/OpenAIClient.js +++ b/api/app/clients/OpenAIClient.js @@ -109,12 +109,7 @@ class OpenAIClient extends BaseClient { const omniPattern = /\b(o1|o3)\b/i; this.isOmni = omniPattern.test(this.modelOptions.model); - const { OPENROUTER_API_KEY, OPENAI_FORCE_PROMPT } = process.env ?? {}; - if (OPENROUTER_API_KEY && !this.azure) { - this.apiKey = OPENROUTER_API_KEY; - this.useOpenRouter = true; - } - + const { OPENAI_FORCE_PROMPT } = process.env ?? {}; const { reverseProxyUrl: reverseProxy } = this.options; if (!this.useOpenRouter && reverseProxy && reverseProxy.includes(KnownEndpoints.openrouter)) { diff --git a/api/app/clients/specs/OpenAIClient.test.js b/api/app/clients/specs/OpenAIClient.test.js index 2aaec518ebe..0e811cf38a4 100644 --- a/api/app/clients/specs/OpenAIClient.test.js +++ b/api/app/clients/specs/OpenAIClient.test.js @@ -202,14 +202,6 @@ describe('OpenAIClient', () => { expect(client.modelOptions.temperature).toBe(0.7); }); - it('should set apiKey and useOpenRouter if OPENROUTER_API_KEY is present', () => { - process.env.OPENROUTER_API_KEY = 'openrouter-key'; - client.setOptions({}); - expect(client.apiKey).toBe('openrouter-key'); - expect(client.useOpenRouter).toBe(true); - delete process.env.OPENROUTER_API_KEY; // Cleanup - }); - it('should set FORCE_PROMPT based on OPENAI_FORCE_PROMPT or reverseProxyUrl', () => { process.env.OPENAI_FORCE_PROMPT = 'true'; client.setOptions({}); @@ -534,7 +526,6 @@ describe('OpenAIClient', () => { afterEach(() => { delete process.env.AZURE_OPENAI_DEFAULT_MODEL; delete process.env.AZURE_USE_MODEL_AS_DEPLOYMENT_NAME; - delete process.env.OPENROUTER_API_KEY; }); it('should call getCompletion and fetchEventSource when using a text/instruct model', async () => { diff --git a/api/server/services/ModelService.js b/api/server/services/ModelService.js index 9630f0bd875..48ae85b6632 100644 --- a/api/server/services/ModelService.js +++ b/api/server/services/ModelService.js @@ -129,9 +129,6 @@ const fetchOpenAIModels = async (opts, _models = []) => { // .split('/deployments')[0] // .concat(`/models?api-version=${azure.azureOpenAIApiVersion}`); // apiKey = azureOpenAIApiKey; - } else if (process.env.OPENROUTER_API_KEY) { - reverseProxyUrl = 'https://openrouter.ai/api/v1'; - apiKey = process.env.OPENROUTER_API_KEY; } if (reverseProxyUrl) { @@ -218,7 +215,7 @@ const getOpenAIModels = async (opts) => { return models; } - if (userProvidedOpenAI && !process.env.OPENROUTER_API_KEY) { + if (userProvidedOpenAI) { return models; } diff --git a/api/server/services/ModelService.spec.js b/api/server/services/ModelService.spec.js index a383db1e3c9..1fbe347a00e 100644 --- a/api/server/services/ModelService.spec.js +++ b/api/server/services/ModelService.spec.js @@ -161,22 +161,6 @@ describe('getOpenAIModels', () => { expect(models).toEqual(expect.arrayContaining(['openai-model', 'openai-model-2'])); }); - it('attempts to use OPENROUTER_API_KEY if set', async () => { - process.env.OPENROUTER_API_KEY = 'test-router-key'; - const expectedModels = ['model-router-1', 'model-router-2']; - - axios.get.mockResolvedValue({ - data: { - data: expectedModels.map((id) => ({ id })), - }, - }); - - const models = await getOpenAIModels({ user: 'user456' }); - - expect(models).toEqual(expect.arrayContaining(expectedModels)); - expect(axios.get).toHaveBeenCalled(); - }); - it('utilizes proxy configuration when PROXY is set', async () => { axios.get.mockResolvedValue({ data: { diff --git a/librechat.example.yaml b/librechat.example.yaml index 637e7a5219b..7913c50455f 100644 --- a/librechat.example.yaml +++ b/librechat.example.yaml @@ -224,7 +224,6 @@ endpoints: - name: 'OpenRouter' # For `apiKey` and `baseURL`, you can use environment variables that you define. # recommended environment variables: - # Known issue: you should not use `OPENROUTER_API_KEY` as it will then override the `openAI` endpoint to use OpenRouter as well. apiKey: '${OPENROUTER_KEY}' baseURL: 'https://openrouter.ai/api/v1' models: From 43b5021ce943966427ed933290f28426c91d94a6 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 26 Feb 2025 13:05:48 -0500 Subject: [PATCH 11/18] refactor: optimize stream chunk handling --- api/app/clients/AnthropicClient.js | 49 ++++++++---------------------- 1 file changed, 12 insertions(+), 37 deletions(-) diff --git a/api/app/clients/AnthropicClient.js b/api/app/clients/AnthropicClient.js index 492def135e7..a2ab752bc22 100644 --- a/api/app/clients/AnthropicClient.js +++ b/api/app/clients/AnthropicClient.js @@ -7,8 +7,7 @@ const { getResponseSender, validateVisionModel, } = require('librechat-data-provider'); -const { SplitStreamHandler, GraphEvents } = require('@librechat/agents'); -const { encodeAndFormat } = require('~/server/services/Files/images/encode'); +const { SplitStreamHandler: _Handler, GraphEvents } = require('@librechat/agents'); const { truncateText, formatMessage, @@ -24,6 +23,7 @@ const { } = require('~/server/services/Endpoints/anthropic/helpers'); const { getModelMaxTokens, getModelMaxOutputTokens, matchModelName } = require('~/utils'); const { spendTokens, spendStructuredTokens } = require('~/models/spendTokens'); +const { encodeAndFormat } = require('~/server/services/Files/images/encode'); const Tokenizer = require('~/server/services/Tokenizer'); const { logger, sendEvent } = require('~/config'); const { sleep } = require('~/server/utils'); @@ -32,6 +32,15 @@ const BaseClient = require('./BaseClient'); const HUMAN_PROMPT = '\n\nHuman:'; const AI_PROMPT = '\n\nAssistant:'; +class SplitStreamHandler extends _Handler { + getDeltaContent(chunk) { + return (chunk?.delta?.text ?? chunk?.completion) || ''; + } + getReasoningDelta(chunk) { + return chunk?.delta?.thinking || ''; + } +} + /** Helper function to introduce a delay before retrying */ function delayBeforeRetry(attempts, baseDelay = 1000) { return new Promise((resolve) => setTimeout(resolve, baseDelay * attempts)); @@ -807,50 +816,16 @@ class AnthropicClient extends BaseClient { } }); - /** @param {string} chunk */ - const handleChunk = (chunk) => { - this.streamHandler.handle({ - choices: [ - { - delta: { - content: chunk, - }, - }, - ], - }); - }; - /** @param {string} chunk */ - const handleReasoningChunk = (chunk) => { - this.streamHandler.handle({ - choices: [ - { - delta: { - reasoning_content: chunk, - }, - }, - ], - }); - }; - for await (const completion of response) { - // Handle each completion as before const type = completion?.type ?? ''; if (tokenEventTypes.has(type)) { logger.debug(`[AnthropicClient] ${type}`, completion); this[type] = completion; } - if (completion?.delta?.thinking) { - handleReasoningChunk(completion.delta.thinking); - } else if (completion?.delta?.text) { - handleChunk(completion.delta.text); - } else if (completion.completion) { - handleChunk(completion.completion); - } - + this.streamHandler.handle(completion); await sleep(streamRate); } - // Successful processing, exit loop break; } catch (error) { attempts += 1; From fd0f34fd305e41c1ae5cc1ddccea0b5009c08b88 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 26 Feb 2025 13:30:46 -0500 Subject: [PATCH 12/18] feat: reset model parameters button --- .../components/SidePanel/Parameters/Panel.tsx | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/client/src/components/SidePanel/Parameters/Panel.tsx b/client/src/components/SidePanel/Parameters/Panel.tsx index 0b9cc364607..00046c3eb5c 100644 --- a/client/src/components/SidePanel/Parameters/Panel.tsx +++ b/client/src/components/SidePanel/Parameters/Panel.tsx @@ -1,3 +1,4 @@ +import { RotateCcw } from 'lucide-react'; import React, { useMemo, useState, useEffect, useCallback } from 'react'; import { getSettingsKeys, tConvoUpdateSchema } from 'librechat-data-provider'; import type { TPreset } from 'librechat-data-provider'; @@ -24,6 +25,7 @@ const excludedKeys = new Set([ '_id', 'tools', 'model', + 'files', ]); export default function Parameters() { @@ -105,6 +107,31 @@ export default function Parameters() { }); }, [parameters, setConversation]); + const resetParameters = useCallback(() => { + setConversation((prev) => { + if (!prev) { + return prev; + } + + const updatedConversation = { ...prev }; + const resetKeys: string[] = []; + + Object.keys(updatedConversation).forEach((key) => { + if (excludedKeys.has(key)) { + return; + } + + if (updatedConversation[key] !== undefined) { + resetKeys.push(key); + delete updatedConversation[key]; + } + }); + + logger.log('parameters', 'parameters reset, affected keys:', resetKeys); + return updatedConversation; + }); + }, [setConversation]); + const openDialog = useCallback(() => { const newPreset = tConvoUpdateSchema.parse({ ...conversation, @@ -146,7 +173,17 @@ export default function Parameters() { ); })} -
+
+ +
+