llama-cpp now in generateObject

package.json

@@ -34,7 +34,7 @@
     "@types/inquirer": "^9.0.7",
     "@types/lodash": "^4.14.202",
     "@types/turndown": "^5.0.4",
-    "ai": "^3.3.7",
+    "ai": "^3.3.19",
     "chrome-paths": "^1.0.1",
     "debug": "^4.3.4",
     "dotenv": "^16.4.4",
@@ -46,7 +46,7 @@
     "lodash": "^4.17.21",
     "node-llama-cpp": "^2.8.12",
     "ollama-ai-provider": "^0.12.0",
-    "openai": "^4.56.0",
+    "openai": "^4.56.1",
     "out-url": "^1.2.2",
     "puppeteer": "^22.0.0",
     "puppeteer-extra": "^3.3.6",

src/agent/config.ts

@@ -1,6 +1,3 @@
-import { createLLMClient } from "llm-polyglot";
-import { ollama } from "ollama-ai-provider";
-
 import {
   ObjectGeneratorOptions,
   DefaultObjectGeneratorOptions,
@@ -12,13 +9,14 @@ export const MinimumResponseTokens = 200;
 export const MaximumResponseTokens = 8_000;
 
 export type ThirdPartyProviders = "openai" | "anthropic";
-export type LocalProviders = "ollama";
+export type LocalProviders = "ollama" | "local";
 export type Providers = ThirdPartyProviders | LocalProviders;
 
 export type ProviderConfig = {
   provider: Providers;
   apiKey: string;
   model: string;
+  path?: string;
 };
 
 export type AgentConfig = {

src/agent/generators/generateObject.ts

@@ -6,6 +6,7 @@ import { ProviderConfig } from "../config";
 
 import { generateObjectOllama } from "./generateObjectOllama";
 import { generateObjectInstructor } from "./generateObjectInstructor";
+import { generateObjectLocal } from "./generateObjectLocal";
 
 // import { generateObjectOpenAI } from "./generateObjectOpenAi";
 
@@ -38,6 +39,12 @@ export async function generateObject<T extends z.ZodSchema<any>>(
     case "ollama": // Ollama has its own function to generate objects because of error handling reasons
       response = await generateObjectOllama(config.model, messages, options);
       break;
+    case "local":
+      if (!config.path) {
+        throw new Error("Local provider requires a path to the model");
+      }
+      response = await generateObjectLocal(config.path, messages, options);
+      break;
     case "openai":
     case "anthropic":
       response = await generateObjectInstructor(config, messages, options);

src/agent/messages.ts

@@ -2,7 +2,6 @@ import { Inventory } from "../inventory";
 import { ObjectiveState, StateType } from "../types/browser";
 import { Memory } from "../types/memory.types";
 import { CoreMessage } from "ai";
-
 /**
  * Stringify an array of objects
  * @param obj - The array of objects to stringify

tests/agent/generators/generateObject.test.ts

@@ -1,8 +1,9 @@
 import { describe, expect, it } from "@jest/globals";
 import { objectiveStateExample1 } from "../../../src/collectiveMemory/examples";
 import { ModelResponseSchema } from "../../../src/types/browser/actionStep.types";
-import { commandPrompt, generateObject } from "../../../src/agent";
+import { commandPrompt, generateObject, getPrompt } from "../../../src/agent";
 import { ProviderConfig } from "../../../src/agent/config";
+import { z } from "zod";
 
 describe("generateObjectLocal", () => {
   it("Should generate an object using the top level function", async () => {
@@ -23,4 +24,25 @@ describe("generateObjectLocal", () => {
 
     expect(parsedRes.command[0].kind).toBeDefined();
   });
+
+  it.skip("Should generate an object using the top level function with local provider", async () => {
+    const ariaTreeExample =
+      '"[0,"RootWebArea","About - High Dimensional Research",[[1,"link","HIGH DIMENSIONAL RESEARCH"],"⚲",[2,"link","NOLITA↗"],[3,"link","MEMORY INDEX"],[4,"link","BLOG"],[5,"link","DOCUMENTATION"],[6,"link","COMPANY"],[7,"heading","We *connect and empower++ new intelligence. "],"WE WANT THE WORLD...","+===+==========================================+\n| 1 | to have a diversity of models,           |\n+---+------------------------------------------+\n| 2 | working as closely together as possible, |\n+---+------------------------------------------+\n| 3 | as productively as possible,             |\n+---+------------------------------------------+\n| 4 | for the betterment of mankind.           |\n+---+------------------------------------------+","We build software for and between models, with a goal of leveraging and coordinating models for autonomous tasks.","",[8,"heading","### TEAM"],[9,"heading","Gates Torrey"],"finance, operations",[10,"link","gates.torrey@hdr.is"],[11,"heading","Tynan Daly"],"product, back-end",[12,"link","tynan.daly@hdr.is"],[13,"heading","Matilde Park"],"product, front-end",[14,"link","matilde.park@hdr.is"],[15,"link","CAREERS →"],[16,"link","JOIN THE MAILING LIST"],[17,"link","DASHBOARD "],"HIGH DIMENSIONAL RESEARCH",[18,"heading","CORE"],[19,"link","NOLITA"],[20,"link","MEMORY INDEX"],[21,"link","BLOG"],[22,"heading","TECHNICAL"],[23,"link","MEMORY API"],[24,"link","GITHUB"],[25,"heading","COMPANY"],[26,"link","PEOPLE"],[27,"link","CAREERS"],[28,"link","TWITTER"],[29,"heading","SUPPORT"],[30,"link","DISCORD"],"SUPPORT@HDR.IS",[31,"link","PRIVACY POLICY"],[32,"link","TERMS OF SERVICE"]]]"';
+    const messages = getPrompt(ariaTreeExample);
+    const providerConfig: ProviderConfig = {
+      provider: "local",
+      model: "null",
+      apiKey: "null",
+      path: "/Users/tynandaly/basin/hdr/browser/capybarahermes-2.5-mistral-7b.Q2_K.gguf",
+    };
+
+    const res = await generateObject(providerConfig, messages, {
+      schema: z.object({ emails: z.array(z.string()) }),
+      name: "ModelResponseSchema",
+      model: "gpt-4-turbo",
+      objectMode: "TOOLS",
+    });
+
+    expect(res.emails).toContain("matilde.park@hdr.is");
+  });
 });