From 0a90278210af52e920685e55db308684d2469b2c Mon Sep 17 00:00:00 2001 From: Maurice Bennett Date: Thu, 10 Aug 2023 18:59:39 -0600 Subject: [PATCH] Add support for weaviate multi tenancy (#2231) * Adding support for weaviate multi-tenancy * removed husky post install script * Reverting changes to package.json * Added clarity around the check for optional tenant when constructing a document * Run lint and format --------- Co-authored-by: jacoblee93 --- langchain/package.json | 4 +-- .../vectorstores/tests/weaviate.int.test.ts | 6 ++-- langchain/src/vectorstores/weaviate.ts | 31 +++++++++++++++---- yarn.lock | 15 +++++++-- 4 files changed, 43 insertions(+), 13 deletions(-) diff --git a/langchain/package.json b/langchain/package.json index 857167dbd70c..2bf5136720bb 100644 --- a/langchain/package.json +++ b/langchain/package.json @@ -651,7 +651,7 @@ "typesense": "^1.5.3", "usearch": "^1.1.1", "vectordb": "^0.1.4", - "weaviate-ts-client": "^1.0.0" + "weaviate-ts-client": "^1.4.0" }, "peerDependencies": { "@aws-sdk/client-dynamodb": "^3.310.0", @@ -716,7 +716,7 @@ "typesense": "^1.5.3", "usearch": "^1.1.1", "vectordb": "^0.1.4", - "weaviate-ts-client": "^1.0.0" + "weaviate-ts-client": "^1.4.0" }, "peerDependenciesMeta": { "@aws-sdk/client-dynamodb": { diff --git a/langchain/src/vectorstores/tests/weaviate.int.test.ts b/langchain/src/vectorstores/tests/weaviate.int.test.ts index a327bba11763..c12e4e53129c 100644 --- a/langchain/src/vectorstores/tests/weaviate.int.test.ts +++ b/langchain/src/vectorstores/tests/weaviate.int.test.ts @@ -5,7 +5,7 @@ import { WeaviateStore } from "../weaviate.js"; import { OpenAIEmbeddings } from "../../embeddings/openai.js"; import { Document } from "../../document.js"; -test.skip("WeaviateStore", async () => { +test("WeaviateStore", async () => { // Something wrong with the weaviate-ts-client types, so we need to disable // eslint-disable-next-line @typescript-eslint/no-explicit-any const client = (weaviate as any).client({ @@ -90,7 +90,7 @@ test.skip("WeaviateStore", async () => { ]); }); -test.skip("WeaviateStore upsert + delete", async () => { +test("WeaviateStore upsert + delete", async () => { // Something wrong with the weaviate-ts-client types, so we need to disable // eslint-disable-next-line @typescript-eslint/no-explicit-any const client = (weaviate as any).client({ @@ -200,7 +200,7 @@ test.skip("WeaviateStore upsert + delete", async () => { ]); }); -test.skip("WeaviateStore delete with filter", async () => { +test("WeaviateStore delete with filter", async () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any const client = (weaviate as any).client({ scheme: diff --git a/langchain/src/vectorstores/weaviate.ts b/langchain/src/vectorstores/weaviate.ts index cc4f0364241a..e8577d5d885b 100644 --- a/langchain/src/vectorstores/weaviate.ts +++ b/langchain/src/vectorstores/weaviate.ts @@ -57,6 +57,7 @@ export interface WeaviateLibArgs { indexName: string; textKey?: string; metadataKeys?: string[]; + tenant?: string; } interface ResultRow { @@ -80,6 +81,8 @@ export class WeaviateStore extends VectorStore { private queryAttrs: string[]; + private tenant?: string; + _vectorstoreType(): string { return "weaviate"; } @@ -91,6 +94,7 @@ export class WeaviateStore extends VectorStore { this.indexName = args.indexName; this.textKey = args.textKey || "text"; this.queryAttrs = [this.textKey]; + this.tenant = args.tenant; if (args.metadataKeys) { this.queryAttrs = [ @@ -126,6 +130,7 @@ export class WeaviateStore extends VectorStore { const flattenedMetadata = flattenObjectForWeaviate(document.metadata); return { + ...(this.tenant ? { tenant: this.tenant } : {}), class: this.indexName, id: documentIds[index], vector: vectors[index], @@ -163,18 +168,28 @@ export class WeaviateStore extends VectorStore { if (ids && ids.length > 0) { for (const id of ids) { - await this.client.data + let deleter = this.client.data .deleter() .withClassName(this.indexName) - .withId(id) - .do(); + .withId(id); + + if (this.tenant) { + deleter = deleter.withTenant(this.tenant); + } + + await deleter.do(); } } else if (filter) { - await this.client.batch + let batchDeleter = this.client.batch .objectsBatchDeleter() .withClassName(this.indexName) - .withWhere(filter.where) - .do(); + .withWhere(filter.where); + + if (this.tenant) { + batchDeleter = batchDeleter.withTenant(this.tenant); + } + + await batchDeleter.do(); } else { throw new Error( `This method requires either "ids" or "filter" to be set in the input object` @@ -198,6 +213,10 @@ export class WeaviateStore extends VectorStore { }) .withLimit(k); + if (this.tenant) { + builder = builder.withTenant(this.tenant); + } + if (filter?.where) { builder = builder.withWhere(filter.where); } diff --git a/yarn.lock b/yarn.lock index db177bf5e440..7f3546a6a761 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13231,7 +13231,7 @@ __metadata: usearch: ^1.1.1 uuid: ^9.0.0 vectordb: ^0.1.4 - weaviate-ts-client: ^1.0.0 + weaviate-ts-client: ^1.4.0 yaml: ^2.2.1 zod: ^3.21.4 zod-to-json-schema: ^3.20.4 @@ -13298,7 +13298,7 @@ __metadata: typesense: ^1.5.3 usearch: ^1.1.1 vectordb: ^0.1.4 - weaviate-ts-client: ^1.0.0 + weaviate-ts-client: ^1.4.0 peerDependenciesMeta: "@aws-sdk/client-dynamodb": optional: true @@ -18992,6 +18992,17 @@ __metadata: languageName: node linkType: hard +"weaviate-ts-client@npm:^1.4.0": + version: 1.4.0 + resolution: "weaviate-ts-client@npm:1.4.0" + dependencies: + graphql-request: ^5.1.0 + isomorphic-fetch: ^3.0.0 + uuid: ^9.0.0 + checksum: aef68e7d77ac8b3d42e1a3dd3c4dcb7a0c74b96c1a3d7b11e52313c7dfa9a9b3fc9ba5f3048628d67193ca6c84bb8a7eca641dacfdc459cfe26ac2fe20b07f11 + languageName: node + linkType: hard + "web-streams-polyfill@npm:4.0.0-beta.3": version: 4.0.0-beta.3 resolution: "web-streams-polyfill@npm:4.0.0-beta.3"