From 94467fae0587a14c18012edd24f33833d13b17e8 Mon Sep 17 00:00:00 2001
From: Adham Badr <adhambadr2@gmail.com>
Date: Sat, 18 Jan 2025 08:48:42 +0000
Subject: [PATCH] feat(anthropic): Add Anthropic PDF support (document type) in
 invoke (#7496)

Co-authored-by: jacoblee93 <jacoblee93@gmail.com>
---
 .../models/chat/integration_anthropic_pdf.ts  | 52 +++++++++++++++++++
 .../src/utils/message_inputs.ts               |  7 +++
 2 files changed, 59 insertions(+)
 create mode 100644 examples/src/models/chat/integration_anthropic_pdf.ts

diff --git a/examples/src/models/chat/integration_anthropic_pdf.ts b/examples/src/models/chat/integration_anthropic_pdf.ts
new file mode 100644
index 000000000000..eef650100b45
--- /dev/null
+++ b/examples/src/models/chat/integration_anthropic_pdf.ts
@@ -0,0 +1,52 @@
+import { ChatAnthropic } from "@langchain/anthropic";
+
+import * as fs from "fs";
+
+export const run = async () => {
+  const llm = new ChatAnthropic({
+    model: "claude-3-5-sonnet-20240620", // Only claude-3-5-sonnet-20240620 , claude-3-5-sonnet-20241022 as of Jan 2025 support PDF documents as in base64
+  });
+
+  // PDF needs to be in Base64.
+  const getLocalFile = async (path: string) => {
+    const localFile = await fs.readFileSync(path);
+    const base64File = localFile.toString("base64");
+    return base64File;
+  };
+
+  // Or remotely
+  const getRemoteFile = async (url: string) => {
+    const response = await fetch(url);
+    const arrayBuffer = await response.arrayBuffer();
+    const base64File = Buffer.from(arrayBuffer).toString("base64");
+    return base64File;
+  };
+
+  const base64 = await getRemoteFile(
+    "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf"
+  );
+
+  const prompt = "Summarise the contents of this PDF";
+
+  const response = await llm.invoke([
+    {
+      role: "user",
+      content: [
+        {
+          type: "text",
+          text: prompt,
+        },
+        {
+          type: "document",
+          source: {
+            media_type: "application/pdf",
+            type: "base64",
+            data: base64,
+          },
+        },
+      ],
+    },
+  ]);
+  console.log(response.content);
+  return response.content;
+};
diff --git a/libs/langchain-anthropic/src/utils/message_inputs.ts b/libs/langchain-anthropic/src/utils/message_inputs.ts
index 0e0be120f864..df44e296901f 100644
--- a/libs/langchain-anthropic/src/utils/message_inputs.ts
+++ b/libs/langchain-anthropic/src/utils/message_inputs.ts
@@ -131,6 +131,13 @@ function _formatContent(content: MessageContent) {
           source,
           ...(cacheControl ? { cache_control: cacheControl } : {}),
         };
+      } else if (contentPart.type === "document") {
+        // PDF
+        return {
+          type: "document",
+          source: contentPart.source,
+          ...(cacheControl ? { cache_control: cacheControl } : {}),
+        };
       } else if (
         textTypes.find((t) => t === contentPart.type) &&
         "text" in contentPart