fix: 4209 - Inconsistent max tokens value persistence

web/containers/ModelDropdown/index.tsx

@@ -99,7 +99,7 @@
   )
 
   const isModelSupportRagAndTools = useCallback((model: Model) => {
     return (
       model?.engine === InferenceEngine.openai ||
       isLocalEngine(model?.engine as InferenceEngine)
     )
@@ -110,7 +110,7 @@
       configuredModels
         .concat(
           downloadedModels.filter(
             (e) => !configuredModels.some((x) => x.id === e.id)
           )
         )
         .filter((e) =>
@@ -120,24 +120,24 @@
           if (searchFilter === 'local') {
             return isLocalEngine(e.engine)
           }
           if (searchFilter === 'remote') {
             return !isLocalEngine(e.engine)
           }
         })
         .sort((a, b) => a.name.localeCompare(b.name))
         .sort((a, b) => {
           const aInDownloadedModels = downloadedModels.some(
             (item) => item.id === a.id
           )
           const bInDownloadedModels = downloadedModels.some(
             (item) => item.id === b.id
           )
           if (aInDownloadedModels && !bInDownloadedModels) {
             return -1
           } else if (!aInDownloadedModels && bInDownloadedModels) {
             return 1
           } else {
             return 0
           }
         }),
     [configuredModels, searchText, searchFilter, downloadedModels]
@@ -145,7 +145,7 @@
 
   useEffect(() => {
     if (open && searchInputRef.current) {
       searchInputRef.current.focus()
     }
   }, [open])
 
@@ -162,9 +162,9 @@
 
   const onClickModelItem = useCallback(
     async (modelId: string) => {
       const model = downloadedModels.find((m) => m.id === modelId)
       setSelectedModel(model)
       setOpen(false)
 
       if (activeThread) {
         // Change assistand tools based on model support RAG
@@ -187,15 +187,19 @@
           ],
         })
 
-        const overriddenSettings =
-          model?.settings.ctx_len && model.settings.ctx_len > 4096
-            ? { ctx_len: 4096 }
-            : {}
+        const defaultContextLength = Math.min(
+          8192,
+          model?.settings.ctx_len ?? 8192
+        )
+        const overriddenParameters = {
+          ctx_len: Math.min(8192, model?.settings.ctx_len ?? 8192),
+          max_tokens: defaultContextLength,
+        }
 
         const modelParams = {
           ...model?.parameters,
           ...model?.settings,
-          ...overriddenSettings,
+          ...overriddenParameters,
         }
 
         // Update model parameter to the thread state

web/hooks/useCreateNewThread.ts

@@ -105,14 +105,21 @@ export const useCreateNewThread = () => {
       enabled: true,
       settings: assistant.tools && assistant.tools[0].settings,
     }
-    const overriddenSettings =
-      defaultModel?.settings.ctx_len && defaultModel.settings.ctx_len > 2048
-        ? { ctx_len: 4096 }
-        : {}
-
-    const overriddenParameters = defaultModel?.parameters.max_tokens
-      ? { max_tokens: 4096 }
-      : {}
+
+    // Default context length is 8192
+    const defaultContextLength = Math.min(
+      8192,
+      defaultModel?.settings.ctx_len ?? 8192
+    )
+
+    const overriddenSettings = {
+      ctx_len: defaultContextLength,
+    }
+
+    // Use ctx length by default
+    const overriddenParameters = {
+      max_tokens: defaultContextLength,
+    }
 
     const createdAt = Date.now()
     let instructions: string | undefined = assistant.instructions