Skip to content

Commit

Permalink
chore: enabling structured type output
Browse files Browse the repository at this point in the history
  • Loading branch information
Casper Bollen authored and Casper Bollen committed Mar 13, 2024
1 parent d8a79cb commit 8c7d9ff
Show file tree
Hide file tree
Showing 3 changed files with 326 additions and 17 deletions.
270 changes: 270 additions & 0 deletions src/Informedica.OpenAI.Lib/Notebooks/StructuredJSON.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,270 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Using Ollama to get structured output"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"dotnet_interactive": {
"language": "fsharp"
},
"polyglot_notebook": {
"kernelName": "fsharp"
}
},
"outputs": [],
"source": [
"#load \"load.fsx\"\n",
"\n",
"open Informedica.OpenAI.Lib\n",
"open Ollama.Operators"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Define a schema and a type for the output\n",
"\n",
"The json function will output the type used as a type parameter. "
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"dotnet_interactive": {
"language": "fsharp"
},
"polyglot_notebook": {
"kernelName": "fsharp"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"ℹ INFO: \n",
"EndPoint: http://localhost:11434/api/chat\n",
"Payload:\n",
"{\"format\":\"json\",\"messages\":[{\"content\":\"\\nUse schema: { number: int; unit: string }\\nWhat is the minimal corrected gestational age mentioned in the text between '''\\n\\n'''A neonate 28 weeks to 32 weeks corrected gestational age.'''\\n\\nReply in JSON.\",\"role\":\"user\"}],\"model\":\"llama2\",\"options\":{\"num_keep\":null,\"seed\":101,\"num_predict\":null,\"top_k\":null,\"top_p\":null,\"tfs_z\":null,\"typical_p\":null,\"repeat_last_n\":64,\"temperature\":0.0,\"repeat_penalty\":null,\"presence_penalty\":null,\"frequency_penalty\":null,\"mirostat\":0,\"mirostat_tau\":null,\"mirostat_eta\":null,\"penalize_newline\":null,\"stop\":[],\"numa\":null,\"num_ctx\":2048,\"num_batch\":null,\"num_gqa\":null,\"num_gpu\":null,\"main_gpu\":null,\"low_vram\":null,\"f16_kv\":null,\"vocab_only\":null,\"use_mmap\":null,\"use_mlock\":null,\"rope_frequency_base\":null,\"rope_frequency_scale\":null,\"num_thread\":null},\"response_format\":{\"schema\":{\n",
" \"$schema\": \"http://json-schema.org/draft-04/schema#\",\n",
" \"title\": \"<>f__AnonymousType3408661278OfIntegerAndString\",\n",
" \"type\": \"object\",\n",
" \"additionalProperties\": false,\n",
" \"properties\": {\n",
" \"number\": {\n",
" \"type\": \"integer\",\n",
" \"format\": \"int32\"\n",
" },\n",
" \"unit\": {\n",
" \"type\": [\n",
" \"null\",\n",
" \"string\"\n",
" ]\n",
" }\n",
" }\n",
"},\"type\":\"json_object\"},\"stream\":false}\n",
"\n",
"{ number = 28\n",
" unit = \"weeks\" }\n"
]
}
],
"source": [
"\n",
"\"\"\"\n",
"Use schema: { number: int; unit: string }\n",
"What is the minimal corrected gestational age mentioned in the text between '''\n",
"\n",
"'''A neonate 28 weeks to 32 weeks corrected gestational age.'''\n",
"\n",
"Reply in JSON.\"\"\"\n",
"|> Message.user\n",
"|> Ollama.json<{| number: int; unit: string |}>\n",
" Ollama.Models.llama2\n",
" []\n",
"|> Async.RunSynchronously\n",
"|> function\n",
" | Ok x -> printfn $\"{x}\"\n",
" | Error e -> printfn $\"oops: {e}\"\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Getting the maximum age as a json structure"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"dotnet_interactive": {
"language": "fsharp"
},
"polyglot_notebook": {
"kernelName": "fsharp"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"ℹ INFO: \n",
"EndPoint: http://localhost:11434/api/chat\n",
"Payload:\n",
"{\"format\":\"json\",\"messages\":[{\"content\":\"\\nUse schema: { number: int; unit: string }\\nWhat is the maximum corrected gestational age mentioned in the text between '''\\n\\n'''A neonate 28 weeks to 32 weeks corrected gestational age.'''\\n\\nReply in JSON.\",\"role\":\"user\"}],\"model\":\"llama2\",\"options\":{\"num_keep\":null,\"seed\":101,\"num_predict\":null,\"top_k\":null,\"top_p\":null,\"tfs_z\":null,\"typical_p\":null,\"repeat_last_n\":64,\"temperature\":0.0,\"repeat_penalty\":null,\"presence_penalty\":null,\"frequency_penalty\":null,\"mirostat\":0,\"mirostat_tau\":null,\"mirostat_eta\":null,\"penalize_newline\":null,\"stop\":[],\"numa\":null,\"num_ctx\":2048,\"num_batch\":null,\"num_gqa\":null,\"num_gpu\":null,\"main_gpu\":null,\"low_vram\":null,\"f16_kv\":null,\"vocab_only\":null,\"use_mmap\":null,\"use_mlock\":null,\"rope_frequency_base\":null,\"rope_frequency_scale\":null,\"num_thread\":null},\"response_format\":{\"schema\":{\n",
" \"$schema\": \"http://json-schema.org/draft-04/schema#\",\n",
" \"title\": \"<>f__AnonymousType3408661278OfIntegerAndString\",\n",
" \"type\": \"object\",\n",
" \"additionalProperties\": false,\n",
" \"properties\": {\n",
" \"number\": {\n",
" \"type\": \"integer\",\n",
" \"format\": \"int32\"\n",
" },\n",
" \"unit\": {\n",
" \"type\": [\n",
" \"null\",\n",
" \"string\"\n",
" ]\n",
" }\n",
" }\n",
"},\"type\":\"json_object\"},\"stream\":false}\n",
"\n",
"{ number = 28\n",
" unit = \"weeks\" }\n"
]
}
],
"source": [
"\"\"\"\n",
"Use schema: { number: int; unit: string }\n",
"What is the maximum corrected gestational age mentioned in the text between '''\n",
"\n",
"'''A neonate 28 weeks to 32 weeks corrected gestational age.'''\n",
"\n",
"Reply in JSON.\"\"\"\n",
"|> Message.user\n",
"|> Ollama.json<{| number: int; unit: string |}>\n",
" Ollama.Models.llama2\n",
" []\n",
"|> Async.RunSynchronously\n",
"|> function\n",
" | Ok x -> printfn $\"{x}\"\n",
" | Error e -> printfn $\"oops: {e}\"\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Try a different structured output"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"dotnet_interactive": {
"language": "fsharp"
},
"polyglot_notebook": {
"kernelName": "fsharp"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"ℹ INFO: \n",
"EndPoint: http://localhost:11434/api/chat\n",
"Payload:\n",
"{\"format\":\"json\",\"messages\":[{\"content\":\"\\nUse schema: { minAge: int; maxAge: int; unit: string }\\nWhat is corrected gestational age range mentioned in the text between '''\\n\\n'''A neonate 28 weeks to 32 weeks corrected gestational age.'''\\n\\nReply in JSON.\",\"role\":\"user\"}],\"model\":\"llama2\",\"options\":{\"num_keep\":null,\"seed\":101,\"num_predict\":null,\"top_k\":null,\"top_p\":null,\"tfs_z\":null,\"typical_p\":null,\"repeat_last_n\":64,\"temperature\":0.0,\"repeat_penalty\":null,\"presence_penalty\":null,\"frequency_penalty\":null,\"mirostat\":0,\"mirostat_tau\":null,\"mirostat_eta\":null,\"penalize_newline\":null,\"stop\":[],\"numa\":null,\"num_ctx\":2048,\"num_batch\":null,\"num_gqa\":null,\"num_gpu\":null,\"main_gpu\":null,\"low_vram\":null,\"f16_kv\":null,\"vocab_only\":null,\"use_mmap\":null,\"use_mlock\":null,\"rope_frequency_base\":null,\"rope_frequency_scale\":null,\"num_thread\":null},\"response_format\":{\"schema\":{\n",
" \"$schema\": \"http://json-schema.org/draft-04/schema#\",\n",
" \"title\": \"<>f__AnonymousType1996685024OfIntegerAndIntegerAndString\",\n",
" \"type\": \"object\",\n",
" \"additionalProperties\": false,\n",
" \"properties\": {\n",
" \"maxAge\": {\n",
" \"type\": \"integer\",\n",
" \"format\": \"int32\"\n",
" },\n",
" \"minAge\": {\n",
" \"type\": \"integer\",\n",
" \"format\": \"int32\"\n",
" },\n",
" \"unit\": {\n",
" \"type\": [\n",
" \"null\",\n",
" \"string\"\n",
" ]\n",
" }\n",
" }\n",
"},\"type\":\"json_object\"},\"stream\":false}\n",
"\n",
"{ maxAge = 32\n",
" minAge = 28\n",
" unit = \"weeks\" }\n"
]
}
],
"source": [
"\"\"\"\n",
"Use schema: { minAge: int; maxAge: int; unit: string }\n",
"What is corrected gestational age range mentioned in the text between '''\n",
"\n",
"'''A neonate 28 weeks to 32 weeks corrected gestational age.'''\n",
"\n",
"Reply in JSON.\"\"\"\n",
"|> Message.user\n",
"|> Ollama.json<{| minAge: int; maxAge: int; unit: string |}>\n",
" Ollama.Models.llama2\n",
" []\n",
"|> Async.RunSynchronously\n",
"|> function\n",
" | Ok x -> printfn $\"{x}\"\n",
" | Error e -> printfn $\"oops: {e}\"\n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": ".NET (C#)",
"language": "C#",
"name": ".net-csharp"
},
"language_info": {
"name": "polyglot-notebook"
},
"polyglot_notebook": {
"kernelInfo": {
"defaultKernelName": "csharp",
"items": [
{
"aliases": [],
"name": "csharp"
},
{
"aliases": [],
"languageName": "fsharp",
"name": "fsharp"
}
]
}
}
},
"nbformat": 4,
"nbformat_minor": 2
}
50 changes: 34 additions & 16 deletions src/Informedica.OpenAI.Lib/Ollama.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,23 +238,41 @@ module Ollama =
content = msg.Content

|}

{|
model = model
format = "json"
response_format = {|
``type`` = "json_object"
schema = "[schema]"
let payload =
{|
model = model
format = "json"
response_format = {|
``type`` = "json_object"
schema = "[schema]"
|}
messages =
[ message |> map ]
|> List.append (messages |> List.map map)
options = options
stream = false
|}
messages =
[ message |> map ]
|> List.append (messages |> List.map map)
options = options
stream = false
|}
|> JsonConvert.SerializeObject
|> String.replace "\"[schema]\"" schema
|> Utils.post<OllamaResponse> EndPoints.chat None
|> JsonConvert.SerializeObject
|> String.replace "\"[schema]\"" schema

async {
let! resp =
payload
|> Utils.post<OllamaResponse> EndPoints.chat None

let resp =
resp
|> Result.bind (fun resp ->
try
resp.Response.message.content
|> JsonConvert.DeserializeObject<'ReturnType>
|> Ok
with
| e -> e.ToString() |> Error
)

return resp
}


let extract tools model messages (message : Message) =
Expand Down
23 changes: 22 additions & 1 deletion src/Informedica.OpenAI.Lib/Scripts/Ollama.fsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,13 +238,34 @@ Ollama.listModels ()

"""
Use schema: { number: int; unit: string }
What is the minimal age for a neonate 28 weeks to 32 weeks corrected gestational age.
What is the minimal corrected gestational age mentioned in the text between '''
'''A neonate 28 weeks to 32 weeks corrected gestational age.'''
Reply just in one JSON."""
|> Message.user
|> Ollama.json<{| number: int; unit: string |}>
Ollama.Models.llama2
[]
|> Async.RunSynchronously
|> function
| Ok x -> printfn $"{x}"
| Error e -> printfn $"oops: {e}"



"""
Use schema: { number: int; unit: string }
Foo bar. Only give an anwer when possible.
Reply just in one JSON."""
|> Message.user
|> Ollama.json<{| number: int; unit: string |}>
Ollama.Models.llama2
[]
|> Async.RunSynchronously
|> function
| Ok x -> printfn $"{x}"
| Error e -> printfn $"oops: {e}"


"""
Expand Down

0 comments on commit 8c7d9ff

Please sign in to comment.