feat: add o3-mini, missing o1 model

dqbd · Feb 2, 2025 · 46e3c7c · 46e3c7c
1 parent 638e6a2
commit 46e3c7c
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 0 deletions.
diff --git a/js/src/core.ts b/js/src/core.ts
@@ -281,11 +281,14 @@ export function getEncodingNameForModel(model: TiktokenModel) {
     case "gpt-4o-2024-08-06":
     case "gpt-4o-mini-2024-07-18":
     case "gpt-4o-mini":
+    case "o1":
     case "o1-2024-12-17":
     case "o1-mini":
     case "o1-preview":
     case "o1-preview-2024-09-12":
     case "o1-mini-2024-09-12":
+    case "o3-mini":
+    case "o3-mini-2025-01-31":
     case "chatgpt-4o-latest":
     case "gpt-4o-realtime":
     case "gpt-4o-realtime-preview-2024-10-01":

diff --git a/tiktoken/model_to_encoding.json b/tiktoken/model_to_encoding.json
@@ -60,11 +60,14 @@
     "gpt-4o-2024-08-06":"o200k_base",
     "gpt-4o-mini-2024-07-18": "o200k_base",
     "gpt-4o-mini": "o200k_base",
+    "o1": "o200k_base",
     "o1-2024-12-17": "o200k_base",
     "o1-mini": "o200k_base",
     "o1-preview": "o200k_base",
     "o1-preview-2024-09-12": "o200k_base",
     "o1-mini-2024-09-12": "o200k_base",
+    "o3-mini": "o200k_base",
+    "o3-mini-2025-01-31": "o200k_base",
     "chatgpt-4o-latest": "o200k_base",
     "gpt-4o-realtime": "o200k_base",
     "gpt-4o-realtime-preview-2024-10-01": "o200k_base"

diff --git a/wasm/src/lib.rs b/wasm/src/lib.rs
@@ -423,11 +423,14 @@ export type TiktokenModel =
     | "gpt-4o-2024-08-06"
     | "gpt-4o-mini-2024-07-18"
     | "gpt-4o-mini"
+    | "o1"
     | "o1-2024-12-17"
     | "o1-mini"
     | "o1-preview"
     | "o1-preview-2024-09-12"
     | "o1-mini-2024-09-12"
+    | "o3-mini"
+    | "o3-mini-2025-01-31"
     | "chatgpt-4o-latest"
     | "gpt-4o-realtime"
     | "gpt-4o-realtime-preview-2024-10-01"