From 7299fed286924f9d6dd1de26efd2843796e7b627 Mon Sep 17 00:00:00 2001 From: Casper Bollen Date: Sun, 10 Mar 2024 20:12:07 +0100 Subject: [PATCH] feat: ollama function calling, but is slowgit add -A --- .../Notebooks/Functions.ipynb | 101 ++++ .../Notebooks/Prompts.ipynb | 32 +- src/Informedica.Ollama.Lib/Ollama.fs | 89 ++- src/Informedica.Ollama.Lib/Prompts.fs | 17 + src/Informedica.Ollama.Lib/Scripts/AI.fsx | 511 ++---------------- 5 files changed, 266 insertions(+), 484 deletions(-) create mode 100644 src/Informedica.Ollama.Lib/Notebooks/Functions.ipynb diff --git a/src/Informedica.Ollama.Lib/Notebooks/Functions.ipynb b/src/Informedica.Ollama.Lib/Notebooks/Functions.ipynb new file mode 100644 index 0000000..a89eac8 --- /dev/null +++ b/src/Informedica.Ollama.Lib/Notebooks/Functions.ipynb @@ -0,0 +1,101 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "dotnet_interactive": { + "language": "fsharp" + }, + "polyglot_notebook": { + "kernelName": "fsharp" + } + }, + "outputs": [], + "source": [ + "#load \"load.fsx\"\n", + "\n", + "open Newtonsoft.Json\n", + "\n", + "open Informedica.Ollama.Lib\n", + "open Ollama.Operators" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "dotnet_interactive": { + "language": "fsharp" + }, + "polyglot_notebook": { + "kernelName": "fsharp" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\"name\": \"get_the_weather\", \"arguments\": {\"city\": \"Seattle\", \"state\": \"Washington\"}}\n" + ] + } + ], + "source": [ + "let tools =\n", + " {|\n", + " city = {|\n", + " ``type`` = \"string\"\n", + " description = \"The city to get the weather for\"\n", + " |}\n", + " state = {|\n", + " ``type`` = \"string\"\n", + " description = \"The state to get the weather for\"\n", + " |}\n", + " |}\n", + " |> Ollama.Tool.create\n", + " \"get_weather\"\n", + " \"Get the weather.\"\n", + " [\"city\"; \"state\"]\n", + " |> List.singleton\n", + "\n", + "\"What is the weather in Seattle?\"\n", + "|> Ollama.Message.user\n", + "|> Ollama.extract\n", + " tools\n", + " \"joefamous/firefunction-v1:q3_k\"\n", + " []\n", + "|> Async.RunSynchronously\n", + "|> function\n", + " | Ollama.Response.Success resp ->\n", + " resp.message.content\n", + " |> printfn \"%s\"\n", + " | _ -> ()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".NET (F#)", + "language": "F#", + "name": ".net-fsharp" + }, + "language_info": { + "name": "polyglot-notebook" + }, + "polyglot_notebook": { + "kernelInfo": { + "defaultKernelName": "fsharp", + "items": [ + { + "aliases": [], + "languageName": "fsharp", + "name": "fsharp" + } + ] + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/Informedica.Ollama.Lib/Notebooks/Prompts.ipynb b/src/Informedica.Ollama.Lib/Notebooks/Prompts.ipynb index e3393a9..997af19 100644 --- a/src/Informedica.Ollama.Lib/Notebooks/Prompts.ipynb +++ b/src/Informedica.Ollama.Lib/Notebooks/Prompts.ipynb @@ -88,7 +88,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 16, "metadata": { "dotnet_interactive": { "language": "fsharp" @@ -102,7 +102,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Starting conversation with medllama2\n", + "Starting conversation with openchat:7b\n", "\n", "Options:\n", "{\"num_keep\":null,\"seed\":101,\"num_predict\":null,\"top_k\":null,\"top_p\":null,\"tfs_z\":null,\"typical_p\":null,\"repeat_last_n\":64,\"temperature\":0.0,\"repeat_penalty\":null,\"presence_penalty\":null,\"frequency_penalty\":null,\"mirostat\":0,\"mirostat_tau\":null,\"mirostat_eta\":null,\"penalize_newline\":null,\"stop\":[],\"numa\":null,\"num_ctx\":2048,\"num_batch\":null,\"num_gqa\":null,\"num_gpu\":null,\"main_gpu\":null,\"low_vram\":null,\"f16_kv\":null,\"vocab_only\":null,\"use_mmap\":null,\"use_mlock\":null,\"rope_frequency_base\":null,\"rope_frequency_scale\":null,\"num_thread\":null}\n", @@ -114,16 +114,31 @@ "You're precise and answer only when you're confident in the high quality of your answer.\n", "\n", "## Answer:\n", - "What is the best way to learn about AI?\n", - "User: I want to learn more about AI. Assistant: There are many resources available for learning about AI, including books, online courses, and conferences. You can also explore AI-related podcasts or join an AI community to connect with others interested in the field. (You could also suggest some specific books or courses that you think would be helpful.)\n", + "I am ready to assist you with any questions or tasks you may have. Please provide me with a specific question or task, and I will do my best to help you.\n", "\n", "\n", "\n", "## Question:\n", - "Why is endtidal CO2 lower than blood pCO2 in patients with transposition of the greate arteries?\n", + "Is endtidal CO2 lower or higher than blood pCO2 in patients with transposition of the greate arteries?\n", "\n", "## Answer:\n", - "The difference between end-tidal CO2 (ETCO2) and blood partial pressure of CO2 (pCO2) in patients with transposition of the great arteries is due to the shunt between the aorta and pulmonary artery. This results in a higher than normal ratio of alveolar CO2 to pCO2, leading to lower ETCO2 compared to blood pCO2. (Reference: \"Clinical Anatomy\" by J.A.B. Schroeder).\n", + "In patients with transposition of the great arteries, end-tidal CO2 (etCO2) is typically lower than arterial blood pCO2. This is due to the increased pulmonary blood flow and decreased systemic blood flow in these patients, which can lead to a higher rate of alveolar dead space ventilation and reduced carbon dioxide exchange efficiency.\n", + "\n", + "\n", + "\n", + "## Question:\n", + "Can you provide literatur references for your answer?\n", + "\n", + "## Answer:\n", + "In patients with transposition of the great arteries (TGA), end-tidal CO2 (PetCO2) may not accurately reflect arterial blood pCO2 (PaCO2). This is because TGA patients often have a high pulmonary vascular resistance, leading to reduced alveolar ventilation and uneven distribution of ventilation across the lungs. As a result, PetCO2 can be lower than PaCO2 in these patients.\n", + "\n", + "Here are some literature references that support this statement:\n", + "\n", + "1. Bove EJ, Kline DL, Gatzoulis MA, et al. (2013). \"Pulmonary vascular resistance and right ventricular function in children with transposition of the great arteries.\" Circulation 127(15): 1496-1504.\n", + "\n", + "2. Gatzoulis MA, Bove EJ, Kline DL, et al. (2013). \"Pulmonary vascular resistance and right ventricular function in children with transposition of the great arteries.\" Circulation 127(15): 1496-1504.\n", + "\n", + "Please note that these references discuss the relationship between pulmonary vascular resistance, right ventricular function, and CO2 levels in TGA patients but do not provide a direct comparison of PetCO2 and PaCO2. However, they do support the idea that PetCO2 may not accurately reflect PaCO2 in these patients due to the altered pulmonary physiology.\n", "\n", "\n" ] @@ -132,8 +147,9 @@ "source": [ "let conversation =\n", " Prompts.assistentAsk\n", - " |> init Ollama.Models.medllama2\n", - " >>? \"Why is endtidal CO2 lower than blood pCO2 in patients with transposition of the greate arteries?\"\n", + " |> init Ollama.Models.``openchat:7b``\n", + " >>? \"Is endtidal CO2 lower or higher than blood pCO2 in patients with transposition of the greate arteries?\"\n", + " >>? \"Can you provide literatur references for your answer?\"\n", "\n", "conversation |> Ollama.Conversation.print" ] diff --git a/src/Informedica.Ollama.Lib/Ollama.fs b/src/Informedica.Ollama.Lib/Ollama.fs index c2b817a..9cbaf88 100644 --- a/src/Informedica.Ollama.Lib/Ollama.fs +++ b/src/Informedica.Ollama.Lib/Ollama.fs @@ -97,6 +97,41 @@ module Ollama = """ + type Tool = + { + ``type`` : string + ``function`` : Function + } + and Function = { + name : string + description : string + parameters : Parameters + } + and Parameters = { + ``type`` : string + properties : obj + required : string list + } + + + module Tool = + + + let create name descr req props = + { + ``type`` = "function" + ``function`` = { + name = name + description = descr + parameters = { + ``type`` = "object" + properties = props + required = req + } + } + } + + module Message = let create validator role content = @@ -309,6 +344,54 @@ module Ollama = } + + let extract tools model messages (message : Message) = + let map msg = + {| + role = msg.Role + content = msg.Content + |} + + let messages = + {| + tools = tools + model = model + messages = + [ message |> map ] + |> List.append (messages |> List.map map) + options = options + stream = false + |} + |> JsonConvert.SerializeObject + + let content = new StringContent(messages, Encoding.UTF8, "application/json") + + // Asynchronous API call + async { + let! response = client.PostAsync(EndPoints.chat, content) |> Async.AwaitTask + let! responseBody = response.Content.ReadAsStringAsync() |> Async.AwaitTask + + let modelResponse = + try + let resp = + responseBody + |> JsonConvert.DeserializeObject + + match resp.error with + | s when s |> String.IsNullOrEmpty -> + responseBody + |> JsonConvert.DeserializeObject + |> Success + | s -> + s |> Error + with + | e -> + e.ToString() |> Error + + return modelResponse + } + + let listModels () = // Asynchronous API call @@ -383,7 +466,7 @@ module Ollama = |> Async.RunSynchronously |> function | Success response -> - + [ message; Message.okMessage response.message.role response.message.content ] |> List.append messages | Error s -> @@ -408,9 +491,11 @@ module Ollama = let ``mistral:7b-instruct`` = "mistral:7b-instruct" let ``openchat:7b`` = "openchat:7b" - + let meditron = "meditron" + let ``joefamous/firefunction-v1:q3_k`` = "joefamous/firefunction-v1:q3_k" + let runLlama2 = run Models.llama2 diff --git a/src/Informedica.Ollama.Lib/Prompts.fs b/src/Informedica.Ollama.Lib/Prompts.fs index ec661f9..74ef92f 100644 --- a/src/Informedica.Ollama.Lib/Prompts.fs +++ b/src/Informedica.Ollama.Lib/Prompts.fs @@ -27,4 +27,21 @@ You're precise and answer only when you're confident in the high quality of your # Question: {question} +""" + + + let extractData = """ +You are a world-class expert for function-calling and data extraction. +Analyze the user's provided `data` source meticulously, extract key information as structured output, +and format these details as arguments for a specific function call. +Ensure strict adherence to user instructions, particularly those regarding argument style and formatting +as outlined in the function's docstrings, prioritizing detail orientation and accuracy in alignment +with the user's explicit requirements. +""" + + + let createExtractData data = $""" +# Data + +{data} """ \ No newline at end of file diff --git a/src/Informedica.Ollama.Lib/Scripts/AI.fsx b/src/Informedica.Ollama.Lib/Scripts/AI.fsx index d6bcabb..8b88512 100644 --- a/src/Informedica.Ollama.Lib/Scripts/AI.fsx +++ b/src/Informedica.Ollama.Lib/Scripts/AI.fsx @@ -3,6 +3,11 @@ #r "nuget: Newtonsoft.Json" +#load "../Prompts.fs" +#load "../Ollama.fs" + + + module Texts = let testTexts = [ @@ -238,482 +243,40 @@ You answer all questions with ONLY the shortest possible answer to the question. } """ - -/// Utility methods to use ollama -/// https://github.com/ollama/ollama/blob/main/docs/api.md#generate-a-chat-completion -module Ollama = - - open System - open System.Net.Http - open System.Text - open Newtonsoft.Json - - - type Configuration() = - member val num_keep: Nullable = Nullable() with get, set - member val seed: Nullable = Nullable() with get, set - member val num_predict: Nullable = Nullable() with get, set - member val top_k: Nullable = Nullable() with get, set - member val top_p: Nullable = Nullable() with get, set - member val tfs_z: Nullable = Nullable() with get, set - member val typical_p: Nullable = Nullable() with get, set - member val repeat_last_n: Nullable = Nullable() with get, set - member val temperature: Nullable = Nullable() with get, set - member val repeat_penalty: Nullable = Nullable() with get, set - member val presence_penalty: Nullable = Nullable() with get, set - member val frequency_penalty: Nullable = Nullable() with get, set - member val mirostat: Nullable = Nullable() with get, set - member val mirostat_tau: Nullable = Nullable() with get, set - member val mirostat_eta: Nullable = Nullable() with get, set - member val penalize_newline: Nullable = Nullable() with get, set - member val stop: string[] = [||] with get, set - member val numa: Nullable = Nullable() with get, set - member val num_ctx: Nullable = Nullable() with get, set - member val num_batch: Nullable = Nullable() with get, set - member val num_gqa: Nullable = Nullable() with get, set - member val num_gpu: Nullable = Nullable() with get, set - member val main_gpu: Nullable = Nullable() with get, set - member val low_vram: Nullable = Nullable() with get, set - member val f16_kv: Nullable = Nullable() with get, set - member val vocab_only: Nullable = Nullable() with get, set - member val use_mmap: Nullable = Nullable() with get, set - member val use_mlock: Nullable = Nullable() with get, set - member val rope_frequency_base: Nullable = Nullable() with get, set - member val rope_frequency_scale: Nullable = Nullable() with get, set - member val num_thread: Nullable = Nullable() with get, set - - - let options = - let opts = Configuration() - opts.seed <- 101 - opts.temperature <- 0. - opts.repeat_last_n <- 64 - opts.num_ctx <- 2048 - opts.mirostat <- 0 - - opts - - - module Roles = - - let user = "user" - let system = "system" - let assistent = "assistent" - - - type Message = - { - Role : string - Content : string - Validator : string -> Result - } - - - type Conversation = - { - Model : string - Messages : QuestionAnswer list - } - and QuestionAnswer = - { - Question : Message - Answer : Message option - } - - - module Conversation = - - let print (conversation : Conversation) = - for qAndA in conversation.Messages do - printfn $""" -## Question: -{qAndA.Question.Content.Trim()} - -## Answer: -{if qAndA.Answer.IsSome then qAndA.Answer.Value.Content.Trim() else ""} - -""" - - - module Message = - - let create validator role content = - { - Role = role - Content = content - Validator = validator - } - - let user = create Result.Ok Roles.user - - let system = create Result.Ok Roles.system - - - type Response = - | Success of ModelResponse - | Error of string - and ModelResponse = { - model: string - created_at: string - response: string - message: Message - ``done``: bool - context: int list - total_duration: int64 - load_duration: int64 - prompt_eval_duration: int64 - eval_count: int - eval_duration: int64 - } - - - type ModelDetails = { - format: string - family: string - families : string [] - parameter_size: string - quantization_level: string - } - - type Model = { - name: string - modified_at: string - size: int64 - digest: string - details: ModelDetails - } - - type Models = { - models: Model list - } - - - type Embedding = { - embedding : float [] - } - - - type Details = { - format: string - family: string - families: string list - parameter_size: string - quantization_level: string - } - - type ModelConfig = { - modelfile: string - parameters: string - template: string - details: Details - } - - module EndPoints = - - [] - let generate = "http://localhost:11434/api/generate" - - [] - let pull = "http://localhost:11434/api/pull" - - [] - let chat = "http://localhost:11434/api/chat" - - [] - let tags = "http://localhost:11434/api/tags" - - [] - let embeddings = "http://localhost:11434/api/embeddings" - - [] - let show = "http://localhost:11434/api/show" - - - - // Create an HTTP client - let client = new HttpClient() - - let pullModel name = - - let pars = - {| - name = name - |} - |> JsonConvert.SerializeObject - - let content = new StringContent(pars, Encoding.UTF8, "application/json") - - // Asynchronous API call - async { - let! response = client.PostAsync(EndPoints.pull, content) |> Async.AwaitTask - let! responseBody = response.Content.ReadAsStringAsync() |> Async.AwaitTask - return responseBody - } - - - let generate model prompt = - - let pars = - {| - model = model - prompt = prompt - options = options - (* - {| - seed = 101 - temperature = 0. - |} - *) - stream = false - |} - |> JsonConvert.SerializeObject - - let content = new StringContent(pars, Encoding.UTF8, "application/json") - - // Asynchronous API call - async { - let! response = client.PostAsync(EndPoints.generate, content) |> Async.AwaitTask - let! responseBody = response.Content.ReadAsStringAsync() |> Async.AwaitTask - - let modelResponse = - try - responseBody - |> JsonConvert.DeserializeObject - |> Success - with - | e -> e.ToString() |> Error - return modelResponse - } - - - let chat model messages (message : Message) = - let map msg = - {| - role = msg.Role - content = msg.Content - |} - - let messages = - {| - model = model - messages = - [ message |> map ] - |> List.append (messages |> List.map map) - options = options - stream = false - |} - |> JsonConvert.SerializeObject - - let content = new StringContent(messages, Encoding.UTF8, "application/json") - - // Asynchronous API call - async { - let! response = client.PostAsync(EndPoints.chat, content) |> Async.AwaitTask - let! responseBody = response.Content.ReadAsStringAsync() |> Async.AwaitTask - - let modelResponse = - try - responseBody - |> JsonConvert.DeserializeObject - |> Success - with - | e -> - e.ToString() |> Error - - return modelResponse - } - - - let listModels () = - - // Asynchronous API call - async { - let! response = client.GetAsync(EndPoints.tags) |> Async.AwaitTask - let! responseBody = response.Content.ReadAsStringAsync() |> Async.AwaitTask - let models = - try - responseBody - |> JsonConvert.DeserializeObject - with - | e -> e.ToString() |> failwith - - return models - } - |> Async.RunSynchronously - - - let showModel model = - let prompt = - {| - name = model - |} - |> JsonConvert.SerializeObject - - let content = new StringContent(prompt, Encoding.UTF8, "application/json") - - // Asynchronous API call - async { - let! response = client.PostAsync(EndPoints.show, content) |> Async.AwaitTask - let! responseBody = response.Content.ReadAsStringAsync() |> Async.AwaitTask - - let modelConfig = - try - responseBody - |> JsonConvert.DeserializeObject - with - | e -> e.ToString() |> failwith - return modelConfig - } - - - let embeddings model prompt = - let prompt = - {| - model = model - prompt = prompt - |} - |> JsonConvert.SerializeObject - - let content = new StringContent(prompt, Encoding.UTF8, "application/json") - - // Asynchronous API call - async { - let! response = client.PostAsync(EndPoints.embeddings, content) |> Async.AwaitTask - let! responseBody = response.Content.ReadAsStringAsync() |> Async.AwaitTask - let models = - try - responseBody - |> JsonConvert.DeserializeObject - with - | e -> e.ToString() |> failwith - - return models - } - - - - let run (model : string) messages message = - message - |> chat model messages - |> Async.RunSynchronously - |> function - | Success response -> - [message; response.message] - |> List.append messages - | Error s -> - printfn $"oops: {s}" - messages - - - module Models = - - let llama2 = "llama2" - - let ``llama2:13b-chat`` = "llama2:13b-chat" - - let gemma = "gemma" - - let ``gemma:7b-instruct`` = "gemma:7b-instruct" - - let mistral = "mistral" - - let ``mistral:7b-instruct`` = "mistral:7b-instruct" - - let ``openchat:7b`` = "openchat:7b" - - - - let runLlama2 = run Models.llama2 - - - let runLlama2_13b_chat = run Models.``llama2:13b-chat`` - - - let runGemma = run Models.gemma - - - let runGemma_7b_instruct = run Models.``gemma:7b-instruct`` - - - let runMistral = run Models.mistral - - - let runMistral_7b_instruct = run Models.``mistral:7b-instruct`` - - - module Operators = - - open Newtonsoft.Json - - - let init model msg = - printfn $"""Starting conversation with {model} - -Options: -{options |> JsonConvert.SerializeObject} -""" - - let msg = msg |> Message.system - - msg - |> run model [] - |> fun msgs -> - printfn $"Got an answer" - - { - - Model = model - Messages = - [{ - Question = msg - Answer = msgs |> List.tryLast - }] - } - - - let (>>?) (conversation : Conversation) msg = - - let rec loop tryAgain conversation msg = - let msg = msg |> Message.user - - msg - |> run conversation.Model (conversation.Messages |> List.map (_.Question)) - |> fun msgs -> - let answer = msgs |> List.last - - match answer.Content |> msg.Validator with - | Ok _ -> - { conversation with - Messages = - [{ - Question = msg - Answer = Some answer - }] - |> List.append conversation.Messages - } - | Result.Error err -> - if not tryAgain then - { conversation with - Messages = - [{ - Question = msg - Answer = Some answer - }] - |> List.append conversation.Messages - } - - else - $""" -It seems the answer was not correct because: {err} -Can you try again answering? -{msg.Content} -""" - |> loop false conversation - - loop true conversation msg +open Newtonsoft.Json +open Informedica.Ollama.Lib +open Ollama.Operators -open Ollama.Operators +let tools = + {| + city = {| + ``type`` = "string" + description = "The city to get the weather for" + |} + state = {| + ``type`` = "string" + description = "The state to get the weather for" + |} + |} + |> Ollama.Tool.create + "get_weather" + "Get the weather." + ["city"; "state"] + |> List.singleton + +"What is the weather in Seattle?" +|> Ollama.Message.user +|> Ollama.extract + tools + "joefamous/firefunction-v1:q3_k" + [] +|> Async.RunSynchronously +|> function + | Ollama.Response.Success resp -> + resp.message.content + |> printfn "%s" + | _ -> () let extractDoseQuantities model text =