chore: add NGL settings for vision models

janhq · Dec 4, 2024 · 68a8293 · 68a8293
1 parent 9e634c7
commit 68a8293
Show file tree

Hide file tree

Showing 4 changed files with 7 additions and 4 deletions.
diff --git a/extensions/inference-cortex-extension/package.json b/extensions/inference-cortex-extension/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@janhq/inference-cortex-extension",
   "productName": "Cortex Inference Engine",
-  "version": "1.0.23",
+  "version": "1.0.24",
   "description": "This extension embeds cortex.cpp, a lightweight inference engine written in C++. See https://jan.ai.\nAdditional dependencies could be installed to run without Cuda Toolkit installation.",
   "main": "dist/index.js",
   "node": "dist/node/index.cjs.js",

diff --git a/extensions/inference-cortex-extension/resources/models/bakllava-1/model.json b/extensions/inference-cortex-extension/resources/models/bakllava-1/model.json
@@ -21,7 +21,8 @@
     "ctx_len": 4096,
     "prompt_template": "\n### Instruction:\n{prompt}\n### Response:\n",
     "llama_model_path": "ggml-model-q5_k.gguf",
-    "mmproj": "mmproj-model-f16.gguf"
+    "mmproj": "mmproj-model-f16.gguf",
+    "ngl": 33
   },
   "parameters": {
     "max_tokens": 4096

diff --git a/extensions/inference-cortex-extension/resources/models/llava-13b/model.json b/extensions/inference-cortex-extension/resources/models/llava-13b/model.json
@@ -21,7 +21,8 @@
     "ctx_len": 4096,
     "prompt_template": "\n### Instruction:\n{prompt}\n### Response:\n",
     "llama_model_path": "llava-v1.6-vicuna-13b.Q4_K_M.gguf",
-    "mmproj": "mmproj-model-f16.gguf"
+    "mmproj": "mmproj-model-f16.gguf",
+    "ngl": 33
   },
   "parameters": {
     "max_tokens": 4096,

diff --git a/extensions/inference-cortex-extension/resources/models/llava-7b/model.json b/extensions/inference-cortex-extension/resources/models/llava-7b/model.json
@@ -21,7 +21,8 @@
     "ctx_len": 4096,
     "prompt_template": "\n### Instruction:\n{prompt}\n### Response:\n",
     "llama_model_path": "llava-v1.6-mistral-7b.Q4_K_M.gguf",
-    "mmproj": "mmproj-model-f16.gguf"
+    "mmproj": "mmproj-model-f16.gguf",
+    "ngl": 33
   },
   "parameters": {
     "max_tokens": 4096,