From ba4eac45d89bcf5aa37fdaf21c1f7f125c73fcb9 Mon Sep 17 00:00:00 2001
From: Jeff Triplett <jeff.triplett@gmail.com>
Date: Sat, 28 Dec 2024 14:18:35 -0600
Subject: [PATCH] :gear: Adds OLLAMA_FLASH_ATTENTION and OLLAMA_KV_CACHE_TYPE

---
 home/.justfiles/ollama.justfile         | 6 ++++++
 home/.plists/homebrew.mxcl.ollama.plist | 3 +++
 2 files changed, 9 insertions(+)

diff --git a/home/.justfiles/ollama.justfile b/home/.justfiles/ollama.justfile
index 98829df..5c54006 100644
--- a/home/.justfiles/ollama.justfile
+++ b/home/.justfiles/ollama.justfile
@@ -5,8 +5,10 @@
 set dotenv-load := false
 set export := true
 
+export OLLAMA_FLASH_ATTENTION := "true"
 export OLLAMA_HOST := "0.0.0.0:11434"
 export OLLAMA_KEEP_ALIVE := "30m"
+export OLLAMA_KV_CACHE_TYPE := "f16"
 export OLLAMA_ORIGINS := "http://*"
 justfile := justfile_directory() + "/.justfiles/ollama.justfile"
 
@@ -55,14 +57,18 @@ justfile := justfile_directory() + "/.justfiles/ollama.justfile"
     ollama list
 
 @getenv:
+    launchctl getenv OLLAMA_FLASH_ATTENTION
     launchctl getenv OLLAMA_HOST
     launchctl getenv OLLAMA_KEEP_ALIVE
+    launchctl getenv OLLAMA_KV_CACHE_TYPE
     launchctl getenv OLLAMA_ORIGINS
 
 @serve *ARGS:
     tandem 'ollama serve {{ ARGS }}'
 
 @setenv:
+    launchctl setenv OLLAMA_FLASH_ATTENTION {{ OLLAMA_FLASH_ATTENTION }}
     launchctl setenv OLLAMA_HOST {{ OLLAMA_HOST }}
     launchctl setenv OLLAMA_KEEP_ALIVE {{ OLLAMA_KEEP_ALIVE }}
+    launchctl setenv OLLAMA_KV_CACHE_TYPE {{ OLLAMA_KV_CACHE_TYPE }}
     launchctl setenv OLLAMA_ORIGINS {{ OLLAMA_ORIGINS }}
diff --git a/home/.plists/homebrew.mxcl.ollama.plist b/home/.plists/homebrew.mxcl.ollama.plist
index 859b98a..84daa73 100644
--- a/home/.plists/homebrew.mxcl.ollama.plist
+++ b/home/.plists/homebrew.mxcl.ollama.plist
@@ -29,6 +29,9 @@
 	<string>/opt/homebrew/var</string>
     <key>EnvironmentVariables</key>
     <dict>
+		<key>OLLAMA_FLASH_ATTENTION</key><string>true</string>
+		<key>OLLAMA_KEEP_ALIVE</key><string>30m</string>
+		<key>OLLAMA_KV_CACHE_TYPE</key><string>f16</string>
         <key>OLLAMA_HOST</key><string>0.0.0.0:11434</string>
         <key>OLLAMA_ORIGINS</key><string>http://*</string>
     </dict>