From 22a1d8645efe02ad4a865111bc8485ee1af40c89 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Mon, 5 Aug 2024 14:49:23 +0800 Subject: [PATCH] .. Signed-off-by: min.tian --- vectordb_bench/__init__.py | 20 +- vectordb_bench/backend/assembler.py | 10 +- vectordb_bench/backend/cases.py | 113 +- vectordb_bench/backend/clients/api.py | 12 +- .../backend/clients/milvus/config.py | 15 +- .../backend/clients/milvus/milvus.py | 133 ++- vectordb_bench/backend/dataset.py | 137 ++- vectordb_bench/backend/filters.py | 68 ++ vectordb_bench/backend/runner/mp_runner.py | 55 +- .../backend/runner/serial_runner.py | 128 ++- vectordb_bench/backend/task_runner.py | 114 +- vectordb_bench/backend/utils.py | 23 +- .../frontend/components/check_results/data.py | 16 +- .../components/check_results/filters.py | 38 +- .../frontend/components/tables/data.py | 7 +- .../frontend/config/dbCaseConfigs.py | 98 +- vectordb_bench/frontend/pages/filter.py | 52 + vectordb_bench/frontend/vdb_benchmark.py | 30 +- vectordb_bench/models.py | 16 +- .../result_20240722_2024072218_milvus.json | 529 +++++++++ .../result_20240723_2024072315_milvus.json | 529 +++++++++ .../result_20240723_2024072316_milvus.json | 493 ++++++++ .../result_20240723_shuffle_milvus.json | 529 +++++++++ .../Milvus/result_20240723_test_milvus.json | 493 ++++++++ .../result_20240725_2024072418_milvus.json | 1015 +++++++++++++++++ vectordb_bench/results/getLeaderboardData.py | 2 +- 26 files changed, 4376 insertions(+), 299 deletions(-) create mode 100644 vectordb_bench/backend/filters.py create mode 100644 vectordb_bench/frontend/pages/filter.py create mode 100644 vectordb_bench/results/Milvus/result_20240722_2024072218_milvus.json create mode 100644 vectordb_bench/results/Milvus/result_20240723_2024072315_milvus.json create mode 100644 vectordb_bench/results/Milvus/result_20240723_2024072316_milvus.json create mode 100644 vectordb_bench/results/Milvus/result_20240723_shuffle_milvus.json create mode 100644 vectordb_bench/results/Milvus/result_20240723_test_milvus.json create mode 100644 vectordb_bench/results/Milvus/result_20240725_2024072418_milvus.json diff --git a/vectordb_bench/__init__.py b/vectordb_bench/__init__.py index f7664502f..118121d46 100644 --- a/vectordb_bench/__init__.py +++ b/vectordb_bench/__init__.py @@ -16,13 +16,14 @@ class config: LOG_LEVEL = env.str("LOG_LEVEL", "INFO") DEFAULT_DATASET_URL = env.str("DEFAULT_DATASET_URL", AWS_S3_URL) - DATASET_LOCAL_DIR = env.path("DATASET_LOCAL_DIR", "/tmp/vectordb_bench/dataset") + # DATASET_LOCAL_DIR = env.path("DATASET_LOCAL_DIR", "/tmp/vectordb_bench/dataset") + DATASET_LOCAL_DIR = env.path("DATASET_LOCAL_DIR", "/home/nas/milvus/tianmin") NUM_PER_BATCH = env.int("NUM_PER_BATCH", 5000) DROP_OLD = env.bool("DROP_OLD", True) - USE_SHUFFLED_DATA = env.bool("USE_SHUFFLED_DATA", True) + USE_SHUFFLED_DATA = env.bool("USE_SHUFFLED_DATA", False) - NUM_CONCURRENCY = env.list("NUM_CONCURRENCY", [1, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100], subcast=int ) + NUM_CONCURRENCY = env.list("NUM_CONCURRENCY", [10], subcast=int) CONCURRENCY_DURATION = 30 @@ -46,14 +47,15 @@ class config: LOAD_TIMEOUT_1536D_500K = 2.5 * 3600 # 2.5h LOAD_TIMEOUT_1536D_5M = 25 * 3600 # 25h - OPTIMIZE_TIMEOUT_DEFAULT = 30 * 60 # 30min - OPTIMIZE_TIMEOUT_768D_1M = 30 * 60 # 30min - OPTIMIZE_TIMEOUT_768D_10M = 5 * 3600 # 5h - OPTIMIZE_TIMEOUT_768D_100M = 50 * 3600 # 50h + OPTIMIZE_TIMEOUT_DEFAULT = 30 * 600 # 30min + OPTIMIZE_TIMEOUT_768D_1M = 30 * 600 # 30min + OPTIMIZE_TIMEOUT_768D_10M = 5 * 36000 # 5h + OPTIMIZE_TIMEOUT_768D_100M = 50 * 36000 # 50h - OPTIMIZE_TIMEOUT_1536D_500K = 15 * 60 # 15min - OPTIMIZE_TIMEOUT_1536D_5M = 2.5 * 3600 # 2.5h + OPTIMIZE_TIMEOUT_1536D_500K = 15 * 600 # 15min + OPTIMIZE_TIMEOUT_1536D_5M = 2.5 * 36000 # 2.5h + def display(self) -> str: tmp = [ i for i in inspect.getmembers(self) diff --git a/vectordb_bench/backend/assembler.py b/vectordb_bench/backend/assembler.py index e7da4d49f..1500d4f54 100644 --- a/vectordb_bench/backend/assembler.py +++ b/vectordb_bench/backend/assembler.py @@ -2,7 +2,7 @@ from .task_runner import CaseRunner, RunningStatus, TaskRunner from ..models import TaskConfig from ..backend.clients import EmptyDBCaseConfig -from ..backend.data_source import DatasetSource +from ..backend.data_source import DatasetSource import logging @@ -11,10 +11,8 @@ class Assembler: @classmethod - def assemble(cls, run_id , task: TaskConfig, source: DatasetSource) -> CaseRunner: - c_cls = task.case_config.case_id.case_cls - - c = c_cls(task.case_config.custom_case) + def assemble(cls, run_id, task: TaskConfig, source: DatasetSource) -> CaseRunner: + c = task.case_config.case if type(task.db_case_config) != EmptyDBCaseConfig: task.db_case_config.metric_type = c.dataset.data.metric_type @@ -55,7 +53,7 @@ def assemble_all( # sort by dataset size for k in db2runner.keys(): - db2runner[k].sort(key=lambda x:x.ca.dataset.data.size) + db2runner[k].sort(key=lambda x: x.ca.dataset.data.size) all_runners = [] all_runners.extend(load_runners) diff --git a/vectordb_bench/backend/cases.py b/vectordb_bench/backend/cases.py index 6c43bb910..0e121b9dc 100644 --- a/vectordb_bench/backend/cases.py +++ b/vectordb_bench/backend/cases.py @@ -1,16 +1,21 @@ -import typing import logging from enum import Enum, auto from typing import Type from vectordb_bench import config from vectordb_bench.backend.clients.api import MetricType +from vectordb_bench.backend.filters import ( + Filter, + FilterType, + IntFilter, + LabelFilter, + NonFilter, +) from vectordb_bench.base import BaseModel from vectordb_bench.frontend.components.custom.getCustomConfig import ( CustomDatasetConfig, ) - -from .dataset import CustomDataset, Dataset, DatasetManager +from .dataset import CustomDataset, Dataset, DatasetManager, DatasetWithSizeType log = logging.getLogger(__name__) @@ -49,6 +54,7 @@ class CaseType(Enum): Custom = 100 PerformanceCustomDataset = 101 + LabelFilterPerformanceCase = 102 def case_cls(self, custom_configs: dict | None = None) -> Type["Case"]: if custom_configs is None: @@ -82,7 +88,8 @@ class Case(BaseModel): label(CaseLabel): performance or load. dataset(DataSet): dataset for this case runner. filter_rate(float | None): one of 99% | 1% | None - filters(dict | None): filters for search + filters(Filter): NonFilter / IntFilter / LabelFilter + with_scalar_labels(bool): whether to insert scalar data (labels) """ case_id: CaseType @@ -90,38 +97,64 @@ class Case(BaseModel): name: str description: str dataset: DatasetManager + with_scalar_labels: bool = False load_timeout: float | int optimize_timeout: float | int | None = None filter_rate: float | None = None + filters: Filter = NonFilter() - @property - def filters(self) -> dict | None: - if self.filter_rate is not None: - ID = round(self.filter_rate * self.dataset.data.size) - return { - "metadata": f">={ID}", - "id": ID, - } + def check_scalar_labels(self) -> None: + if self.with_scalar_labels and not self.dataset.data.with_scalar_labels: + raise ValueError( + f"""no scalar_labels data in current dataset ({self.dataset.data.name})""" + ) + if self.filters.type == FilterType.Label and not self.with_scalar_labels: + raise ValueError("label-filter cases need scalar_labels data") + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + # post check + self.check_scalar_labels() + + # @property + # def filters(self) -> dict | None: + # if self.filter_rate is not None: + # ID = round(self.filter_rate * self.dataset.data.size) + # return { + # "metadata": f">={ID}", + # "id": ID, + # } - return None + # return None -class CapacityCase(Case, BaseModel): +class CapacityCase(Case): label: CaseLabel = CaseLabel.Load filter_rate: float | None = None load_timeout: float | int = config.CAPACITY_TIMEOUT_IN_SECONDS optimize_timeout: float | int | None = None -class PerformanceCase(Case, BaseModel): +class PerformanceCase(Case): label: CaseLabel = CaseLabel.Performance filter_rate: float | None = None load_timeout: float | int = config.LOAD_TIMEOUT_DEFAULT optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_DEFAULT +class IntFilterPerformanceCase(PerformanceCase): + @property + def filters(self) -> Filter: + int_field = "id" + int_value = self.dataset.data.size * self.filter_rate + return IntFilter( + filter_rate=self.filter_rate, int_field=int_field, int_value=int_value + ) + + class CapacityDim960(CapacityCase): case_id: CaseType = CaseType.CapacityDim960 dataset: DatasetManager = Dataset.GIST.manager(100_000) @@ -160,7 +193,7 @@ class Performance768D1M(PerformanceCase): optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_768D_1M -class Performance768D10M1P(PerformanceCase): +class Performance768D10M1P(IntFilterPerformanceCase): case_id: CaseType = CaseType.Performance768D10M1P filter_rate: float | int | None = 0.01 dataset: DatasetManager = Dataset.COHERE.manager(10_000_000) @@ -171,7 +204,7 @@ class Performance768D10M1P(PerformanceCase): optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_768D_10M -class Performance768D1M1P(PerformanceCase): +class Performance768D1M1P(IntFilterPerformanceCase): case_id: CaseType = CaseType.Performance768D1M1P filter_rate: float | int | None = 0.01 dataset: DatasetManager = Dataset.COHERE.manager(1_000_000) @@ -182,7 +215,7 @@ class Performance768D1M1P(PerformanceCase): optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_768D_1M -class Performance768D10M99P(PerformanceCase): +class Performance768D10M99P(IntFilterPerformanceCase): case_id: CaseType = CaseType.Performance768D10M99P filter_rate: float | int | None = 0.99 dataset: DatasetManager = Dataset.COHERE.manager(10_000_000) @@ -193,7 +226,7 @@ class Performance768D10M99P(PerformanceCase): optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_768D_10M -class Performance768D1M99P(PerformanceCase): +class Performance768D1M99P(IntFilterPerformanceCase): case_id: CaseType = CaseType.Performance768D1M99P filter_rate: float | int | None = 0.99 dataset: DatasetManager = Dataset.COHERE.manager(1_000_000) @@ -237,7 +270,7 @@ class Performance1536D5M(PerformanceCase): optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1536D_5M -class Performance1536D500K1P(PerformanceCase): +class Performance1536D500K1P(IntFilterPerformanceCase): case_id: CaseType = CaseType.Performance1536D500K1P filter_rate: float | int | None = 0.01 dataset: DatasetManager = Dataset.OPENAI.manager(500_000) @@ -248,7 +281,7 @@ class Performance1536D500K1P(PerformanceCase): optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1536D_500K -class Performance1536D5M1P(PerformanceCase): +class Performance1536D5M1P(IntFilterPerformanceCase): case_id: CaseType = CaseType.Performance1536D5M1P filter_rate: float | int | None = 0.01 dataset: DatasetManager = Dataset.OPENAI.manager(5_000_000) @@ -259,7 +292,7 @@ class Performance1536D5M1P(PerformanceCase): optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1536D_5M -class Performance1536D500K99P(PerformanceCase): +class Performance1536D500K99P(IntFilterPerformanceCase): case_id: CaseType = CaseType.Performance1536D500K99P filter_rate: float | int | None = 0.99 dataset: DatasetManager = Dataset.OPENAI.manager(500_000) @@ -270,7 +303,7 @@ class Performance1536D500K99P(PerformanceCase): optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1536D_500K -class Performance1536D5M99P(PerformanceCase): +class Performance1536D5M99P(IntFilterPerformanceCase): case_id: CaseType = CaseType.Performance1536D5M99P filter_rate: float | int | None = 0.99 dataset: DatasetManager = Dataset.OPENAI.manager(5_000_000) @@ -304,6 +337,39 @@ def metric_type_map(s: str) -> MetricType: raise RuntimeError(err_msg) +class LabelFilterPerformanceCase(PerformanceCase): + case_id: CaseType = CaseType.LabelFilterPerformanceCase + with_scalar_labels: bool = True + + def __init__( + self, + dataset_with_size_type: DatasetWithSizeType, + label_percentage: float, + **kwargs, + ): + if not isinstance(dataset_with_size_type, DatasetWithSizeType): + dataset_with_size_type = DatasetWithSizeType(dataset_with_size_type) + name = ( + f"Label-Filter-{label_percentage*100:.1f}% - {dataset_with_size_type.value}" + ) + description = f"Label-Filter-{label_percentage*100:.1f}% Performance Test ({dataset_with_size_type.value})" + dataset = dataset_with_size_type.get_manager() + load_timeout = dataset_with_size_type.get_load_timeout() + optimize_timeout = dataset_with_size_type.get_optimize_timeout() + filter = LabelFilter(label_percentage=label_percentage) + filter_rate = filter.filter_rate + super().__init__( + name=name, + description=description, + dataset=dataset, + load_timeout=load_timeout, + optimize_timeout=optimize_timeout, + filter_rate=filter_rate, + filters=filter, + **kwargs, + ) + + class PerformanceCustomDataset(PerformanceCase): case_id: CaseType = CaseType.PerformanceCustomDataset name: str = "Performance With Custom Dataset" @@ -357,4 +423,5 @@ def __init__( CaseType.Performance1536D5M99P: Performance1536D5M99P, CaseType.Performance1536D50K: Performance1536D50K, CaseType.PerformanceCustomDataset: PerformanceCustomDataset, + CaseType.LabelFilterPerformanceCase: LabelFilterPerformanceCase, } diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index d9ec5d83b..774dcd67f 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -5,6 +5,8 @@ from pydantic import BaseModel, validator, SecretStr +from vectordb_bench.backend.filters import Filter + class MetricType(str, Enum): L2 = "L2" @@ -167,13 +169,11 @@ def insert_embeddings( """ raise NotImplementedError + def prepare_filters(self, filters: Filter): + pass + @abstractmethod - def search_embedding( - self, - query: list[float], - k: int = 100, - filters: dict | None = None, - ) -> list[int]: + def search_embedding(self, query: list[float], k: int = 100, **kwargs) -> list[int]: """Get k most similar embeddings to query vector. Args: diff --git a/vectordb_bench/backend/clients/milvus/config.py b/vectordb_bench/backend/clients/milvus/config.py index eea7b11f5..eaf08e202 100644 --- a/vectordb_bench/backend/clients/milvus/config.py +++ b/vectordb_bench/backend/clients/milvus/config.py @@ -3,7 +3,7 @@ class MilvusConfig(DBConfig): - uri: SecretStr = "http://localhost:19530" + uri: SecretStr = "http://10.104.22.89:19530" def to_dict(self) -> dict: return {"uri": self.uri.get_secret_value()} @@ -14,10 +14,14 @@ class MilvusIndexConfig(BaseModel): index: IndexType metric_type: MetricType | None = None - + @property def is_gpu_index(self) -> bool: - return self.index in [IndexType.GPU_CAGRA, IndexType.GPU_IVF_FLAT, IndexType.GPU_IVF_PQ] + return self.index in [ + IndexType.GPU_CAGRA, + IndexType.GPU_IVF_FLAT, + IndexType.GPU_IVF_PQ, + ] def parse_metric(self) -> str: if not self.metric_type: @@ -99,7 +103,8 @@ def search_param(self) -> dict: "metric_type": self.parse_metric(), "params": {"nprobe": self.nprobe}, } - + + class IVFSQ8Config(MilvusIndexConfig, DBCaseConfig): nlist: int nprobe: int | None = None @@ -196,7 +201,7 @@ class GPUCAGRAConfig(MilvusIndexConfig, DBCaseConfig): search_width: int = 4 min_iterations: int = 0 max_iterations: int = 0 - build_algo: str = "IVF_PQ" # IVF_PQ; NN_DESCENT; + build_algo: str = "IVF_PQ" # IVF_PQ; NN_DESCENT; cache_dataset_on_device: str refine_ratio: float | None = None index: IndexType = IndexType.GPU_CAGRA diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index 4590265ae..2002dd985 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -4,17 +4,18 @@ import time from contextlib import contextmanager from typing import Iterable - from pymilvus import Collection, utility from pymilvus import CollectionSchema, DataType, FieldSchema, MilvusException -from ..api import VectorDB, IndexType +from vectordb_bench.backend.filters import Filter, FilterType +from ..api import VectorDB from .config import MilvusIndexConfig log = logging.getLogger(__name__) -MILVUS_LOAD_REQS_SIZE = 1.5 * 1024 *1024 +MILVUS_LOAD_REQS_SIZE = 1.5 * 1024 * 1024 + class Milvus(VectorDB): def __init__( @@ -25,6 +26,7 @@ def __init__( collection_name: str = "VectorDBBenchCollection", drop_old: bool = False, name: str = "Milvus", + with_scalar_labels: bool = False, **kwargs, ): """Initialize wrapper around the milvus vector database.""" @@ -32,14 +34,19 @@ def __init__( self.db_config = db_config self.case_config = db_case_config self.collection_name = collection_name - self.batch_size = int(MILVUS_LOAD_REQS_SIZE / (dim *4)) + self.batch_size = int(MILVUS_LOAD_REQS_SIZE / (dim * 4)) + self.with_scalar_labels = with_scalar_labels self._primary_field = "pk" - self._scalar_field = "id" + self._scalar_id_field = "id" + self._scalar_label_field = "label" self._vector_field = "vector" - self._index_name = "vector_idx" + self._vector_index_name = "vector_idx" + self._scalar_id_index_name = "id_sort_idx" + self._scalar_labels_index_name = "labels_idx" from pymilvus import connections + connections.connect(**self.db_config, timeout=30) if drop_old and utility.has_collection(self.collection_name): log.info(f"{self.name} client drop_old collection: {self.collection_name}") @@ -48,9 +55,19 @@ def __init__( if not utility.has_collection(self.collection_name): fields = [ FieldSchema(self._primary_field, DataType.INT64, is_primary=True), - FieldSchema(self._scalar_field, DataType.INT64), - FieldSchema(self._vector_field, DataType.FLOAT_VECTOR, dim=dim) + FieldSchema(self._scalar_id_field, DataType.INT64), + FieldSchema(self._vector_field, DataType.FLOAT_VECTOR, dim=dim), ] + if self.with_scalar_labels: + log.info("with_scalar_labels, add a new varchar field") + fields.append( + FieldSchema( + self._scalar_label_field, + DataType.VARCHAR, + max_length=32, + is_partition_key=True, + ) + ) log.info(f"{self.name} create collection: {self.collection_name}") @@ -61,15 +78,39 @@ def __init__( consistency_level="Session", ) - col.create_index( - self._vector_field, - self.case_config.index_param(), - index_name=self._index_name, - ) + self.create_index() + # self._pre_load(coll) connections.disconnect("default") + def create_index(self, only_vector_index=False): + col = Collection(self.collection_name) + # vector index + col.create_index( + self._vector_field, + self.case_config.index_param(), + index_name=self._vector_index_name, + ) + if not only_vector_index: + # int index for range-expr + col.create_index( + self._scalar_id_field, + index_params={ + "index_type": "STL_SORT", + }, + index_name=self._scalar_id_index_name, + ) + if self.with_scalar_labels: + # inverted index for varchar + col.create_index( + self._scalar_label_field, + index_params={ + "index_type": "INVERTED", + }, + index_name=self._scalar_labels_index_name, + ) + @contextmanager def init(self) -> None: """ @@ -79,6 +120,7 @@ def init(self) -> None: >>> self.search_embedding() """ from pymilvus import connections + self.col: Collection | None = None connections.connect(**self.db_config, timeout=60) @@ -102,16 +144,17 @@ def _post_insert(self): try: self.col.flush() # wait for index done and load refresh - self.col.create_index( - self._vector_field, - self.case_config.index_param(), - index_name=self._index_name, + self.create_index() + + utility.wait_for_index_building_complete( + self.collection_name, index_name=self._vector_index_name ) - utility.wait_for_index_building_complete(self.collection_name) def wait_index(): while True: - progress = utility.index_building_progress(self.collection_name) + progress = utility.index_building_progress( + self.collection_name, index_name=self._vector_index_name + ) if progress.get("pending_index_rows", -1) == 0: break time.sleep(5) @@ -121,14 +164,14 @@ def wait_index(): # Skip compaction if use GPU indexType if self.case_config.is_gpu_index: log.debug("skip compaction for gpu index type.") - else : + else: try: self.col.compact() self.col.wait_for_compaction_completed() except Exception as e: log.warning(f"{self.name} compact error: {e}") - if hasattr(e, 'code'): - if e.code().name == 'PERMISSION_DENIED': + if hasattr(e, "code"): + if e.code().name == "PERMISSION_DENIED": log.warning(f"Skip compact due to permission denied.") pass else: @@ -142,22 +185,17 @@ def ready_to_load(self): assert self.col, "Please call self.init() before" self._pre_load(self.col) - def _pre_load(self, coll: Collection): + def _pre_load(self, col: Collection): try: - if not coll.has_index(index_name=self._index_name): + if not col.has_index(index_name=self._vector_index_name): log.info(f"{self.name} create index") - coll.create_index( - self._vector_field, - self.case_config.index_param(), - index_name=self._index_name, - ) - coll.load() + self.create_index(only_vector_index=True) + col.load() log.info(f"{self.name} load") except Exception as e: log.warning(f"{self.name} pre load error: {e}") raise e from None - def optimize(self): assert self.col, "Please call self.init() before" self._optimize() @@ -165,7 +203,9 @@ def optimize(self): def need_normalize_cosine(self) -> bool: """Wheather this database need to normalize dataset to support COSINE""" if self.case_config.is_gpu_index: - log.info(f"current gpu_index only supports IP / L2, cosine dataset need normalize.") + log.info( + "current gpu_index only supports IP / L2, cosine dataset need normalize." + ) return True return False @@ -174,6 +214,7 @@ def insert_embeddings( self, embeddings: Iterable[list[float]], metadata: list[int], + labels_data: list[str], **kwargs, ) -> (int, Exception): """Insert embeddings into Milvus. should call self.init() first""" @@ -183,11 +224,14 @@ def insert_embeddings( insert_count = 0 try: for batch_start_offset in range(0, len(embeddings), self.batch_size): - batch_end_offset = min(batch_start_offset + self.batch_size, len(embeddings)) + batch_end_offset = min( + batch_start_offset + self.batch_size, len(embeddings) + ) insert_data = [ - metadata[batch_start_offset : batch_end_offset], - metadata[batch_start_offset : batch_end_offset], - embeddings[batch_start_offset : batch_end_offset], + metadata[batch_start_offset:batch_end_offset], + metadata[batch_start_offset:batch_end_offset], + embeddings[batch_start_offset:batch_end_offset], + labels_data[batch_start_offset:batch_end_offset], ] res = self.col.insert(insert_data) insert_count += len(res.primary_keys) @@ -196,25 +240,32 @@ def insert_embeddings( return (insert_count, e) return (insert_count, None) + def prepare_filters(self, filters: Filter): + if filters.type == FilterType.NonFilter: + self.expr = "" + elif filters.type == FilterType.Int: + self.expr = f"{self._scalar_id_field} >= {filters.int_value}" + elif filters.type == FilterType.Label: + self.expr = f"{self._scalar_label_field} == '{filters.label_value}'" + else: + raise RuntimeError(f"Not support Filter - {filters}") + def search_embedding( self, query: list[float], - k: int = 100, - filters: dict | None = None, + k: int, timeout: int | None = None, ) -> list[int]: """Perform a search on a query embedding and return results.""" assert self.col is not None - expr = f"{self._scalar_field} {filters.get('metadata')}" if filters else "" - # Perform the search. res = self.col.search( data=[query], anns_field=self._vector_field, param=self.case_config.search_param(), limit=k, - expr=expr, + expr=self.expr, ) # Organize results. diff --git a/vectordb_bench/backend/dataset.py b/vectordb_bench/backend/dataset.py index d559eb6be..af5df2fd6 100644 --- a/vectordb_bench/backend/dataset.py +++ b/vectordb_bench/backend/dataset.py @@ -13,6 +13,8 @@ import polars as pl from pyarrow.parquet import ParquetFile +from vectordb_bench.backend.filters import Filter + from ..base import BaseModel from .. import config from ..backend.clients import MetricType @@ -22,7 +24,7 @@ log = logging.getLogger(__name__) -SizeLabel = namedtuple('SizeLabel', ['size', 'label', 'file_count']) +SizeLabel = namedtuple("SizeLabel", ["size", "label", "file_count"]) class BaseDataset(BaseModel): @@ -34,11 +36,15 @@ class BaseDataset(BaseModel): with_gt: bool = False _size_label: dict[int, SizeLabel] = PrivateAttr() isCustom: bool = False + with_scalar_labels: bool = False + scalar_label_percentages: list[float] = [] @validator("size") def verify_size(cls, v): if v not in cls._size_label: - raise ValueError(f"Size {v} not supported for the dataset, expected: {cls._size_label.keys()}") + raise ValueError( + f"Size {v} not supported for the dataset, expected: {cls._size_label.keys()}" + ) return v @property @@ -53,15 +59,16 @@ def dir_name(self) -> str: def file_count(self) -> int: return self._size_label.get(self.size).file_count + class CustomDataset(BaseDataset): dir: str file_num: int isCustom: bool = True - + @validator("size") def verify_size(cls, v): return v - + @property def label(self) -> str: return "Custom" @@ -73,7 +80,8 @@ def dir_name(self) -> str: @property def file_count(self) -> int: return self.file_num - + + class LAION(BaseDataset): name: str = "LAION" dim: int = 768 @@ -101,12 +109,24 @@ class Cohere(BaseDataset): dim: int = 768 metric_type: MetricType = MetricType.COSINE use_shuffled: bool = config.USE_SHUFFLED_DATA - with_gt: bool = True, + with_gt: bool = (True,) _size_label: dict = { 100_000: SizeLabel(100_000, "SMALL", 1), 1_000_000: SizeLabel(1_000_000, "MEDIUM", 1), 10_000_000: SizeLabel(10_000_000, "LARGE", 10), } + with_scalar_labels: bool = True + scalar_label_percentages: list[float] = [ + 0.001, + 0.002, + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.2, + 0.5, + ] class Glove(BaseDataset): @@ -123,7 +143,11 @@ class SIFT(BaseDataset): metric_type: MetricType = MetricType.L2 use_shuffled: bool = False _size_label: dict = { - 500_000: SizeLabel(500_000, "SMALL", 1,), + 500_000: SizeLabel( + 500_000, + "SMALL", + 1, + ), 5_000_000: SizeLabel(5_000_000, "MEDIUM", 1), # 50_000_000: SizeLabel(50_000_000, "LARGE", 50), } @@ -134,12 +158,24 @@ class OpenAI(BaseDataset): dim: int = 1536 metric_type: MetricType = MetricType.COSINE use_shuffled: bool = config.USE_SHUFFLED_DATA - with_gt: bool = True, + with_gt: bool = (True,) _size_label: dict = { 50_000: SizeLabel(50_000, "SMALL", 1), 500_000: SizeLabel(500_000, "MEDIUM", 1), 5_000_000: SizeLabel(5_000_000, "LARGE", 10), } + with_scalar_labels: bool = True + scalar_label_percentages: list[float] = [ + 0.001, + 0.002, + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.2, + 0.5, + ] class DatasetManager(BaseModel): @@ -152,10 +188,12 @@ class DatasetManager(BaseModel): >>> for data in cohere: >>> print(data.columns) """ - data: BaseDataset + + data: BaseDataset test_data: pd.DataFrame | None = None gt_data: pd.DataFrame | None = None - train_files : list[str] = [] + scalar_labels: pd.DataFrame | None = None + train_files: list[str] = [] reader: DatasetReader | None = None def __eq__(self, obj): @@ -168,22 +206,26 @@ def set_reader(self, reader: DatasetReader): @property def data_dir(self) -> pathlib.Path: - """ data local directory: config.DATASET_LOCAL_DIR/{dataset_name}/{dataset_dirname} + """data local directory: config.DATASET_LOCAL_DIR/{dataset_name}/{dataset_dirname} Examples: >>> sift_s = Dataset.SIFT.manager(500_000) >>> sift_s.relative_path '/tmp/vectordb_bench/dataset/sift/sift_small_500k/' """ - return pathlib.Path(config.DATASET_LOCAL_DIR, self.data.name.lower(), self.data.dir_name.lower()) + # return pathlib.Path( + # config.DATASET_LOCAL_DIR, self.data.name.lower(), self.data.dir_name.lower() + # ) + return pathlib.Path(config.DATASET_LOCAL_DIR, self.data.dir_name.lower()) def __iter__(self): return DataSetIterator(self) # TODO passing use_shuffle from outside - def prepare(self, - source: DatasetSource=DatasetSource.S3, - filters: int | float | str | None = None, + def prepare( + self, + filters: Filter, + source: DatasetSource = DatasetSource.S3, ) -> bool: """Download the dataset from DatasetSource url = f"{source}/{self.data.dir_name}" @@ -202,24 +244,34 @@ def prepare(self, train_files = utils.compose_train_files(file_count, use_shuffled) all_files = train_files + scalar_labels_file = None + if self.data.with_scalar_labels: + scalar_labels_file = "scalar_labels.parquet" + all_files.append(scalar_labels_file) + gt_file, test_file = None, None if self.data.with_gt: - gt_file, test_file = utils.compose_gt_file(filters), "test.parquet" + gt_file, test_file = filters.get_groundtruth_file(), "test.parquet" all_files.extend([gt_file, test_file]) - if not self.data.isCustom: - source.reader().read( - dataset=self.data.dir_name.lower(), - files=all_files, - local_ds_root=self.data_dir, - ) + # if not self.data.isCustom: + # source.reader().read( + # dataset=self.data.dir_name.lower(), + # files=all_files, + # local_ds_root=self.data_dir, + # ) + + if scalar_labels_file is not None: + self.scalar_labels = self._read_file(scalar_labels_file) if gt_file is not None and test_file is not None: self.test_data = self._read_file(test_file) self.gt_data = self._read_file(gt_file) prefix = "shuffle_train" if use_shuffled else "train" - self.train_files = sorted([f.name for f in self.data_dir.glob(f'{prefix}*.parquet')]) + self.train_files = sorted( + [f.name for f in self.data_dir.glob(f"{prefix}*.parquet")] + ) log.debug(f"{self.data.name}: available train files {self.train_files}") return True @@ -240,7 +292,9 @@ def __init__(self, dataset: DatasetManager): self._ds = dataset self._idx = 0 # file number self._cur = None - self._sub_idx = [0 for i in range(len(self._ds.train_files))] # iter num for each file + self._sub_idx = [ + 0 for i in range(len(self._ds.train_files)) + ] # iter num for each file def _get_iter(self, file_name: str): p = pathlib.Path(self._ds.data_dir, file_name) @@ -278,6 +332,7 @@ class Dataset(Enum): >>> Dataset.COHERE.manager(100_000) >>> Dataset.COHERE.get(100_000) """ + LAION = LAION GIST = GIST COHERE = Cohere @@ -290,3 +345,37 @@ def get(self, size: int) -> BaseDataset: def manager(self, size: int) -> DatasetManager: return DatasetManager(data=self.get(size)) + + +class DatasetWithSizeType(Enum): + CohereMedium = "Medium Cohere (768dim, 1M)" + CohereLarge = "Large Cohere (768dim, 10M)" + OpenAIMedium = "Medium OpenAI (1536dim, 500K)" + OpenAILarge = "Large OpenAI (1536dim, 5M)" + + def get_manager(self) -> DatasetManager: + if self not in DatasetWithSizeMap: + raise ValueError(f"wrong ScalarDatasetWithSizeType: {self.name}") + return DatasetWithSizeMap.get(self) + + def get_load_timeout(self) -> float: + if "medium" in self.value.lower(): + return config.LOAD_TIMEOUT_768D_1M + if "large" in self.value.lower(): + return config.LOAD_TIMEOUT_768D_10M + raise config.LOAD_TIMEOUT_DEFAULT + + def get_optimize_timeout(self) -> float: + if "medium" in self.value.lower(): + return config.OPTIMIZE_TIMEOUT_768D_1M + if "large" in self.value.lower(): + return config.OPTIMIZE_TIMEOUT_768D_10M + raise config.OPTIMIZE_TIMEOUT_DEFAULT + + +DatasetWithSizeMap = { + DatasetWithSizeType.CohereMedium: Dataset.COHERE.manager(1_000_000), + DatasetWithSizeType.CohereLarge: Dataset.COHERE.manager(10_000_000), + DatasetWithSizeType.OpenAIMedium: Dataset.OPENAI.manager(500_000), + DatasetWithSizeType.OpenAILarge: Dataset.OPENAI.manager(5_000_000), +} diff --git a/vectordb_bench/backend/filters.py b/vectordb_bench/backend/filters.py new file mode 100644 index 000000000..317532a66 --- /dev/null +++ b/vectordb_bench/backend/filters.py @@ -0,0 +1,68 @@ +from ..base import BaseModel +from enum import StrEnum + + +class FilterType(StrEnum): + Int = "Int" # test ">=" + Label = "Label" # test "==" + NonFilter = "NonFilter" + + +class Filter(BaseModel): + type: FilterType + filter_rate: float = 0.0 + + def get_groundtruth_file(self) -> str: + raise NotImplementedError + + +class NonFilter(Filter): + type: FilterType = FilterType.NonFilter + filter_rate: float = 0.0 + + def get_groundtruth_file(self) -> str: + return "neighbors.parquet" + + +class IntFilter(Filter): + """ + filter expr: id >= count * filter_rate + """ + + type: FilterType = FilterType.Int + int_field: str = "id" + int_value: int + + def get_groundtruth_file(self) -> str: + if self.filter_rate == 0.01: + return "neighbors_head_1p.parquet" + if self.filter_rate == 0.99: + return "neighbors_tail_1p.parquet" + raise RuntimeError(f"Not Support Int Filter - {self.filter_rate}") + + +class LabelFilter(Filter): + """ + filter expr: label_field == label_value, like `color == "red"` + """ + + type: FilterType = FilterType.Label + label_field: str = "labels" + label_percentage: float + + @property + def label_value(self) -> str: + p = self.label_percentage * 100 + if p >= 1: + return f"label_{int(p)}p" # such as 5p, 20p, 1p, ... + else: + return f"label_{p:.1f}p" # such as 0.1p, 0.5p, ... + + def __init__(self, label_percentage: float, **kwargs): + filter_rate = 1.0 - label_percentage + super().__init__( + filter_rate=filter_rate, label_percentage=label_percentage, **kwargs + ) + + def get_groundtruth_file(self) -> str: + return f"neighbors_{self.label_field}_{self.label_value}.parquet" diff --git a/vectordb_bench/backend/runner/mp_runner.py b/vectordb_bench/backend/runner/mp_runner.py index 3b99cfe6e..caaa0f1e1 100644 --- a/vectordb_bench/backend/runner/mp_runner.py +++ b/vectordb_bench/backend/runner/mp_runner.py @@ -5,6 +5,8 @@ import logging from typing import Iterable import numpy as np + +from vectordb_bench.backend.filters import Filter from ..clients import api from ... import config @@ -14,19 +16,20 @@ class MultiProcessingSearchRunner: - """ multiprocessing search runner + """multiprocessing search runner Args: k(int): search topk, default to 100 concurrency(Iterable): concurrencies, default [1, 5, 10, 15, 20, 25, 30, 35] duration(int): duration for each concurency, default to 30s """ + def __init__( self, db: api.VectorDB, test_data: list[list[float]], - k: int = 100, - filters: dict | None = None, + k: int, + filters: Filter, concurrencies: Iterable[int] = config.NUM_CONCURRENCY, duration: int = 30, ): @@ -39,13 +42,16 @@ def __init__( self.test_data = test_data log.debug(f"test dataset columns: {len(test_data)}") - def search(self, test_data: list[list[float]], q: mp.Queue, cond: mp.Condition) -> tuple[int, float]: + def search( + self, test_data: list[list[float]], q: mp.Queue, cond: mp.Condition + ) -> tuple[int, float]: # sync all process q.put(1) with cond: cond.wait() with self.db.init(): + self.db.prepare_filters(self.filters) num, idx = len(test_data), 0 start_time = time.perf_counter() @@ -63,27 +69,31 @@ def search(self, test_data: list[list[float]], q: mp.Queue, cond: mp.Condition) log.warning(f"VectorDB search_embedding error: {e}") traceback.print_exc(chain=True) raise e from None - + latencies.append(time.perf_counter() - s) count += 1 # loop through the test data idx = idx + 1 if idx < num - 1 else 0 if count % 500 == 0: - log.debug(f"({mp.current_process().name:16}) search_count: {count}, latest_latency={time.perf_counter()-s}") + log.debug( + f"({mp.current_process().name:16}) search_count: {count}, latest_latency={time.perf_counter()-s}" + ) total_dur = round(time.perf_counter() - start_time, 4) log.info( f"{mp.current_process().name:16} search {self.duration}s: " f"actual_dur={total_dur}s, count={count}, qps in this process: {round(count / total_dur, 4):3}" - ) + ) return (count, total_dur, latencies) @staticmethod def get_mp_context(): mp_start_method = "spawn" - log.debug(f"MultiProcessingSearchRunner get multiprocessing start method: {mp_start_method}") + log.debug( + f"MultiProcessingSearchRunner get multiprocessing start method: {mp_start_method}" + ) return mp.get_context(mp_start_method) def _run_all_concurrencies_mem_efficient(self) -> float: @@ -95,9 +105,16 @@ def _run_all_concurrencies_mem_efficient(self) -> float: for conc in self.concurrencies: with mp.Manager() as m: q, cond = m.Queue(), m.Condition() - with concurrent.futures.ProcessPoolExecutor(mp_context=self.get_mp_context(), max_workers=conc) as executor: - log.info(f"Start search {self.duration}s in concurrency {conc}, filters: {self.filters}") - future_iter = [executor.submit(self.search, self.test_data, q, cond) for i in range(conc)] + with concurrent.futures.ProcessPoolExecutor( + mp_context=self.get_mp_context(), max_workers=conc + ) as executor: + log.info( + f"Start search {self.duration}s in concurrency {conc}, filters: {self.filters}" + ) + future_iter = [ + executor.submit(self.search, self.test_data, q, cond) + for i in range(conc) + ] # Sync all processes while q.qsize() < conc: sleep_t = conc if conc < 10 else 10 @@ -105,7 +122,9 @@ def _run_all_concurrencies_mem_efficient(self) -> float: with cond: cond.notify_all() - log.info(f"Syncing all process and start concurrency search, concurrency={conc}") + log.info( + f"Syncing all process and start concurrency search, concurrency={conc}" + ) start = time.perf_counter() all_count = sum([r.result()[0] for r in future_iter]) @@ -117,13 +136,19 @@ def _run_all_concurrencies_mem_efficient(self) -> float: conc_num_list.append(conc) conc_qps_list.append(qps) conc_latency_p99_list.append(latency_p99) - log.info(f"End search in concurrency {conc}: dur={cost}s, total_count={all_count}, qps={qps}") + log.info( + f"End search in concurrency {conc}: dur={cost}s, total_count={all_count}, qps={qps}" + ) if qps > max_qps: max_qps = qps - log.info(f"Update largest qps with concurrency {conc}: current max_qps={max_qps}") + log.info( + f"Update largest qps with concurrency {conc}: current max_qps={max_qps}" + ) except Exception as e: - log.warning(f"Fail to search all concurrencies: {self.concurrencies}, max_qps before failure={max_qps}, reason={e}") + log.warning( + f"Fail to search all concurrencies: {self.concurrencies}, max_qps before failure={max_qps}, reason={e}" + ) traceback.print_exc() # No results available, raise exception diff --git a/vectordb_bench/backend/runner/serial_runner.py b/vectordb_bench/backend/runner/serial_runner.py index 9e6818443..d844ae983 100644 --- a/vectordb_bench/backend/runner/serial_runner.py +++ b/vectordb_bench/backend/runner/serial_runner.py @@ -9,6 +9,8 @@ import numpy as np import pandas as pd +from vectordb_bench.backend.filters import Filter + from ..clients import api from ...metric import calc_ndcg, calc_recall, get_ideal_dcg from ...models import LoadTimeoutError, PerformanceTimeoutError @@ -22,8 +24,15 @@ log = logging.getLogger(__name__) + class SerialInsertRunner: - def __init__(self, db: api.VectorDB, dataset: DatasetManager, normalize: bool, timeout: float | None = None): + def __init__( + self, + db: api.VectorDB, + dataset: DatasetManager, + normalize: bool, + timeout: float | None = None, + ): self.timeout = timeout if isinstance(timeout, (int, float)) else None self.dataset = dataset self.db = db @@ -32,23 +41,34 @@ def __init__(self, db: api.VectorDB, dataset: DatasetManager, normalize: bool, t def task(self) -> int: count = 0 with self.db.init(): - log.info(f"({mp.current_process().name:16}) Start inserting embeddings in batch {config.NUM_PER_BATCH}") + log.info( + f"({mp.current_process().name:16}) Start inserting embeddings in batch {config.NUM_PER_BATCH}" + ) start = time.perf_counter() for data_df in self.dataset: - all_metadata = data_df['id'].tolist() + all_metadata = data_df["id"].tolist() - emb_np = np.stack(data_df['emb']) + emb_np = np.stack(data_df["emb"]) if self.normalize: log.debug("normalize the 100k train data") - all_embeddings = (emb_np / np.linalg.norm(emb_np, axis=1)[:, np.newaxis]).tolist() + all_embeddings = ( + emb_np / np.linalg.norm(emb_np, axis=1)[:, np.newaxis] + ).tolist() else: all_embeddings = emb_np.tolist() - del(emb_np) - log.debug(f"batch dataset size: {len(all_embeddings)}, {len(all_metadata)}") + del emb_np + log.debug( + f"batch dataset size: {len(all_embeddings)}, {len(all_metadata)}" + ) + + labels_data = self.dataset.scalar_labels["labels"][ + all_metadata + ].to_list() insert_count, error = self.db.insert_embeddings( embeddings=all_embeddings, metadata=all_metadata, + labels_data=labels_data, ) if error is not None: raise error @@ -56,30 +76,44 @@ def task(self) -> int: assert insert_count == len(all_metadata) count += insert_count if count % 100_000 == 0: - log.info(f"({mp.current_process().name:16}) Loaded {count} embeddings into VectorDB") + log.info( + f"({mp.current_process().name:16}) Loaded {count} embeddings into VectorDB" + ) - log.info(f"({mp.current_process().name:16}) Finish loading all dataset into VectorDB, dur={time.perf_counter()-start}") + log.info( + f"({mp.current_process().name:16}) Finish loading all dataset into VectorDB, dur={time.perf_counter()-start}" + ) return count - def endless_insert_data(self, all_embeddings, all_metadata, left_id: int = 0) -> int: + def endless_insert_data( + self, all_embeddings, all_metadata, left_id: int = 0 + ) -> int: with self.db.init(): # unique id for endlessness insertion - all_metadata = [i+left_id for i in all_metadata] + all_metadata = [i + left_id for i in all_metadata] - NUM_BATCHES = math.ceil(len(all_embeddings)/NUM_PER_BATCH) - log.info(f"({mp.current_process().name:16}) Start inserting {len(all_embeddings)} embeddings in batch {NUM_PER_BATCH}") + NUM_BATCHES = math.ceil(len(all_embeddings) / NUM_PER_BATCH) + log.info( + f"({mp.current_process().name:16}) Start inserting {len(all_embeddings)} embeddings in batch {NUM_PER_BATCH}" + ) count = 0 for batch_id in range(NUM_BATCHES): retry_count = 0 already_insert_count = 0 - metadata = all_metadata[batch_id*NUM_PER_BATCH : (batch_id+1)*NUM_PER_BATCH] - embeddings = all_embeddings[batch_id*NUM_PER_BATCH : (batch_id+1)*NUM_PER_BATCH] - - log.debug(f"({mp.current_process().name:16}) batch [{batch_id:3}/{NUM_BATCHES}], Start inserting {len(metadata)} embeddings") + metadata = all_metadata[ + batch_id * NUM_PER_BATCH : (batch_id + 1) * NUM_PER_BATCH + ] + embeddings = all_embeddings[ + batch_id * NUM_PER_BATCH : (batch_id + 1) * NUM_PER_BATCH + ] + + log.debug( + f"({mp.current_process().name:16}) batch [{batch_id:3}/{NUM_BATCHES}], Start inserting {len(metadata)} embeddings" + ) while retry_count < LOAD_MAX_TRY_COUNT: insert_count, error = self.db.insert_embeddings( - embeddings=embeddings[already_insert_count :], - metadata=metadata[already_insert_count :], + embeddings=embeddings[already_insert_count:], + metadata=metadata[already_insert_count:], ) already_insert_count += insert_count if error is not None: @@ -91,17 +125,23 @@ def endless_insert_data(self, all_embeddings, all_metadata, left_id: int = 0) -> raise error else: break - log.debug(f"({mp.current_process().name:16}) batch [{batch_id:3}/{NUM_BATCHES}], Finish inserting {len(metadata)} embeddings") + log.debug( + f"({mp.current_process().name:16}) batch [{batch_id:3}/{NUM_BATCHES}], Finish inserting {len(metadata)} embeddings" + ) assert already_insert_count == len(metadata) count += already_insert_count - log.info(f"({mp.current_process().name:16}) Finish inserting {len(all_embeddings)} embeddings in batch {NUM_PER_BATCH}") + log.info( + f"({mp.current_process().name:16}) Finish inserting {len(all_embeddings)} embeddings in batch {NUM_PER_BATCH}" + ) return count @utils.time_it def _insert_all_batches(self) -> int: """Performance case only""" - with concurrent.futures.ProcessPoolExecutor(mp_context=mp.get_context('spawn'), max_workers=1) as executor: + with concurrent.futures.ProcessPoolExecutor( + mp_context=mp.get_context("spawn"), max_workers=1 + ) as executor: future = executor.submit(self.task) try: count = future.result(timeout=self.timeout) @@ -122,7 +162,10 @@ def run_endlessness(self) -> int: # datasets for load tests are quite small, can fit into memory # only 1 file data_df = [data_df for data_df in self.dataset][0] - all_embeddings, all_metadata = np.stack(data_df["emb"]).tolist(), data_df['id'].tolist() + all_embeddings, all_metadata = ( + np.stack(data_df["emb"]).tolist(), + data_df["id"].tolist(), + ) start_time = time.perf_counter() max_load_count, times = 0, 0 @@ -130,12 +173,18 @@ def run_endlessness(self) -> int: with self.db.init(): self.db.ready_to_load() while time.perf_counter() - start_time < self.timeout: - count = self.endless_insert_data(all_embeddings, all_metadata, left_id=max_load_count) + count = self.endless_insert_data( + all_embeddings, all_metadata, left_id=max_load_count + ) max_load_count += count times += 1 - log.info(f"Loaded {times} entire dataset, current max load counts={utils.numerize(max_load_count)}, {max_load_count}") + log.info( + f"Loaded {times} entire dataset, current max load counts={utils.numerize(max_load_count)}, {max_load_count}" + ) except Exception as e: - log.info(f"Capacity case load reach limit, insertion counts={utils.numerize(max_load_count)}, {max_load_count}, err={e}") + log.info( + f"Capacity case load reach limit, insertion counts={utils.numerize(max_load_count)}, {max_load_count}, err={e}" + ) traceback.print_exc() return max_load_count else: @@ -154,8 +203,8 @@ def __init__( db: api.VectorDB, test_data: list[list[float]], ground_truth: pd.DataFrame, - k: int = 100, - filters: dict | None = None, + k: int, + filters: Filter, ): self.db = db self.k = k @@ -168,13 +217,18 @@ def __init__( self.ground_truth = ground_truth def search(self, args: tuple[list, pd.DataFrame]): - log.info(f"{mp.current_process().name:14} start search the entire test_data to get recall and latency") + log.info( + f"{mp.current_process().name:14} start search the entire test_data to get recall and latency" + ) with self.db.init(): + self.db.prepare_filters(self.filters) test_data, ground_truth = args ideal_dcg = get_ideal_dcg(self.k) log.debug(f"test dataset size: {len(test_data)}") - log.debug(f"ground truth size: {ground_truth.columns}, shape: {ground_truth.shape}") + log.debug( + f"ground truth size: {ground_truth.columns}, shape: {ground_truth.shape}" + ) latencies, recalls, ndcgs = [], [], [] for idx, emb in enumerate(test_data): @@ -193,13 +247,14 @@ def search(self, args: tuple[list, pd.DataFrame]): latencies.append(time.perf_counter() - s) - gt = ground_truth['neighbors_id'][idx] - recalls.append(calc_recall(self.k, gt[:self.k], results)) - ndcgs.append(calc_ndcg(gt[:self.k], results, ideal_dcg)) - + gt = ground_truth["neighbors_id"][idx] + recalls.append(calc_recall(self.k, gt[: self.k], results)) + ndcgs.append(calc_ndcg(gt[: self.k], results, ideal_dcg)) if len(latencies) % 100 == 0: - log.debug(f"({mp.current_process().name:14}) search_count={len(latencies):3}, latest_latency={latencies[-1]}, latest recall={recalls[-1]}") + log.debug( + f"({mp.current_process().name:14}) search_count={len(latencies):3}, latest_latency={latencies[-1]}, latest recall={recalls[-1]}" + ) avg_latency = round(np.mean(latencies), 4) avg_recall = round(np.mean(recalls), 4) @@ -214,10 +269,9 @@ def search(self, args: tuple[list, pd.DataFrame]): f"avg_ndcg={avg_ndcg}," f"avg_latency={avg_latency}, " f"p99={p99}" - ) + ) return (avg_recall, avg_ndcg, p99) - def _run_in_subprocess(self) -> tuple[float, float]: with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: future = executor.submit(self.search, (self.test_data, self.ground_truth)) diff --git a/vectordb_bench/backend/task_runner.py b/vectordb_bench/backend/task_runner.py index a6d94f186..eca5e466e 100644 --- a/vectordb_bench/backend/task_runner.py +++ b/vectordb_bench/backend/task_runner.py @@ -10,14 +10,11 @@ from ..base import BaseModel from ..models import TaskConfig, PerformanceTimeoutError, TaskStage -from .clients import ( - api, - MetricType -) +from .clients import api, MetricType from ..metric import Metric from .runner import MultiProcessingSearchRunner from .runner import SerialSearchRunner, SerialInsertRunner -from .data_source import DatasetSource +from .data_source import DatasetSource log = logging.getLogger(__name__) @@ -55,22 +52,41 @@ class CaseRunner(BaseModel): def __eq__(self, obj): if isinstance(obj, CaseRunner): - return self.ca.label == CaseLabel.Performance and \ - self.config.db == obj.config.db and \ - self.config.db_case_config == obj.config.db_case_config and \ - self.ca.dataset == obj.ca.dataset + return ( + self.ca.label == CaseLabel.Performance + and self.config.db == obj.config.db + and self.config.db_case_config == obj.config.db_case_config + and self.ca.dataset == obj.ca.dataset + and self.ca.with_scalar_labels == self.ca.with_scalar_labels + ) return False def display(self) -> dict: - c_dict = self.ca.dict(include={'label':True, 'filters': True,'dataset':{'data': {'name': True, 'size': True, 'dim': True, 'metric_type': True, 'label': True}} }) - c_dict['db'] = self.config.db_name + c_dict = self.ca.dict( + include={ + "label": True, + "filters": True, + "dataset": { + "data": { + "name": True, + "size": True, + "dim": True, + "metric_type": True, + "label": True, + } + }, + } + ) + c_dict["db"] = self.config.db_name return c_dict @property def normalize(self) -> bool: assert self.db - return self.db.need_normalize_cosine() and \ - self.ca.dataset.data.metric_type == MetricType.COSINE + return ( + self.db.need_normalize_cosine() + and self.ca.dataset.data.metric_type == MetricType.COSINE + ) def init_db(self, drop_old: bool = True) -> None: db_cls = self.config.db.init_cls @@ -80,13 +96,13 @@ def init_db(self, drop_old: bool = True) -> None: db_config=self.config.db_config.to_dict(), db_case_config=self.config.db_case_config, drop_old=drop_old, - ) # type:ignore - + with_scalar_labels=self.ca.with_scalar_labels, + ) def _pre_run(self, drop_old: bool = True): try: self.init_db(drop_old) - self.ca.dataset.prepare(self.dataset_source, filters=self.ca.filter_rate) + self.ca.dataset.prepare(source=self.dataset_source, filters=self.ca.filters) except ModuleNotFoundError as e: log.warning( f"pre run case error: please install client for db: {self.config.db}, error={e}" @@ -138,7 +154,7 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: Returns: Metric: load_duration, recall, serial_latency_p99, and, qps """ - ''' + """ if drop_old: _, load_dur = self._load_train_data() build_dur = self._optimize() @@ -153,7 +169,7 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: m.qps, m.conc_num_list, m.conc_qps_list, m.conc_latency_p99_list = self._conc_search() m.recall, m.serial_latency_p99 = self._serial_search() - ''' + """ log.info("Start performance case") try: @@ -178,15 +194,20 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: self._init_search_runner() if TaskStage.SEARCH_SERIAL in self.config.stages: search_results = self._serial_search() - ''' + """ m.recall = search_results.recall m.serial_latencies = search_results.serial_latencies - ''' + """ m.recall, m.ndcg, m.serial_latency_p99 = search_results if TaskStage.SEARCH_CONCURRENT in self.config.stages: search_results = self._conc_search() - m.qps, m.conc_num_list, m.conc_qps_list, m.conc_latency_p99_list = search_results - + ( + m.qps, + m.conc_num_list, + m.conc_qps_list, + m.conc_latency_p99_list, + ) = search_results + except Exception as e: log.warning(f"Failed to run performance case, reason = {e}") traceback.print_exc() @@ -199,7 +220,9 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: def _load_train_data(self): """Insert train data and get the insert_duration""" try: - runner = SerialInsertRunner(self.db, self.ca.dataset, self.normalize, self.ca.load_timeout) + runner = SerialInsertRunner( + self.db, self.ca.dataset, self.normalize, self.ca.load_timeout + ) runner.run() except Exception as e: raise e from None @@ -249,7 +272,9 @@ def _optimize(self) -> float: log.warning(f"VectorDB optimize timeout in {self.ca.optimize_timeout}") for pid, _ in executor._processes.items(): psutil.Process(pid).kill() - raise PerformanceTimeoutError("Performance case optimize timeout") from e + raise PerformanceTimeoutError( + "Performance case optimize timeout" + ) from e except Exception as e: log.warning(f"VectorDB optimize error: {e}") raise e from None @@ -303,35 +328,32 @@ def _get_num_by_status(self, status: RunningStatus) -> int: return sum([1 for c in self.case_runners if c.status == status]) def display(self) -> None: - DATA_FORMAT = (" %-14s | %-12s %-20s %7s | %-10s") + DATA_FORMAT = " %-14s | %-12s %-20s %7s | %-10s" TITLE_FORMAT = (" %-14s | %-12s %-20s %7s | %-10s") % ( - "DB", "CaseType", "Dataset", "Filter", "task_label") + "DB", + "CaseType", + "Dataset", + "Filter", + "task_label", + ) fmt = [TITLE_FORMAT] - fmt.append(DATA_FORMAT%( - "-"*11, - "-"*12, - "-"*20, - "-"*7, - "-"*7 - )) + fmt.append(DATA_FORMAT % ("-" * 11, "-" * 12, "-" * 20, "-" * 7, "-" * 7)) for f in self.case_runners: - if f.ca.filter_rate != 0.0: - filters = f.ca.filter_rate - elif f.ca.filter_size != 0: - filters = f.ca.filter_size - else: - filters = "None" + filter_rate = f.ca.filter_rate ds_str = f"{f.ca.dataset.data.name}-{f.ca.dataset.data.label}-{utils.numerize(f.ca.dataset.data.size)}" - fmt.append(DATA_FORMAT%( - f.config.db_name, - f.ca.label.name, - ds_str, - filters, - self.task_label, - )) + fmt.append( + DATA_FORMAT + % ( + f.config.db_name, + f.ca.label.name, + ds_str, + filter_rate, + self.task_label, + ) + ) tmp_logger = logging.getLogger("no_color") for f in fmt: diff --git a/vectordb_bench/backend/utils.py b/vectordb_bench/backend/utils.py index 690cff3e8..101ce8071 100644 --- a/vectordb_bench/backend/utils.py +++ b/vectordb_bench/backend/utils.py @@ -16,20 +16,20 @@ def numerize(n) -> str: "K": 1e6, "M": 1e9, "B": 1e12, - "END": float('inf'), + "END": float("inf"), } display_n, sufix = n, "" for s, base in sufix2upbound.items(): # number >= 1000B will alway have sufix 'B' if s == "END": - display_n = int(n/1e9) + display_n = int(n / 1e9) sufix = "B" break if n < base: sufix = "" if s == "EMPTY" else s - display_n = int(n/(base/1e3)) + display_n = int(n / (base / 1e3)) break return f"{display_n}{sufix}" @@ -41,6 +41,7 @@ def inner(*args, **kwargs): result = func(*args, **kwargs) delta = time.perf_counter() - pref return result, delta + return inner @@ -61,14 +62,14 @@ def compose_train_files(train_count: int, use_shuffled: bool) -> list[str]: return train_files -def compose_gt_file(filters: int | float | str | None = None) -> str: - if filters is None: - return "neighbors.parquet" +# def compose_gt_file(filters: int | float | str | None = None) -> str: +# if filters is None: +# return "neighbors.parquet" - if filters == 0.01: - return "neighbors_head_1p.parquet" +# if filters == 0.01: +# return "neighbors_head_1p.parquet" - if filters == 0.99: - return "neighbors_tail_1p.parquet" +# if filters == 0.99: +# return "neighbors_tail_1p.parquet" - raise ValueError(f"Filters not supported: {filters}") +# raise ValueError(f"Filters not supported: {filters}") diff --git a/vectordb_bench/frontend/components/check_results/data.py b/vectordb_bench/frontend/components/check_results/data.py index b3cac21e1..5a2e28944 100644 --- a/vectordb_bench/frontend/components/check_results/data.py +++ b/vectordb_bench/frontend/components/check_results/data.py @@ -1,6 +1,5 @@ from collections import defaultdict from dataclasses import asdict -from vectordb_bench.backend.cases import Case from vectordb_bench.metric import isLowerIsBetterMetric from vectordb_bench.models import CaseResult, ResultLabel @@ -24,10 +23,7 @@ def getFilterTasks( task for task in tasks if task.task_config.db_name in dbNames - and task.task_config.case_config.case_id.case_cls( - task.task_config.case_config.custom_case - ).name - in caseNames + and task.task_config.case_config.case_name in caseNames ] return filterTasks @@ -39,19 +35,17 @@ def mergeTasks(tasks: list[CaseResult]): db = task.task_config.db.value db_label = task.task_config.db_config.db_label or "" version = task.task_config.db_config.version or "" - case = task.task_config.case_config.case_id.case_cls( - task.task_config.case_config.custom_case - ) - dbCaseMetricsMap[db_name][case.name] = { + case_name = task.task_config.case_config.case_name + dbCaseMetricsMap[db_name][case_name] = { "db": db, "db_label": db_label, "version": version, "metrics": mergeMetrics( - dbCaseMetricsMap[db_name][case.name].get("metrics", {}), + dbCaseMetricsMap[db_name][case_name].get("metrics", {}), asdict(task.metrics), ), "label": getBetterLabel( - dbCaseMetricsMap[db_name][case.name].get("label", ResultLabel.FAILED), + dbCaseMetricsMap[db_name][case_name].get("label", ResultLabel.FAILED), task.label, ), } diff --git a/vectordb_bench/frontend/components/check_results/filters.py b/vectordb_bench/frontend/components/check_results/filters.py index e60efb2e1..34da854bb 100644 --- a/vectordb_bench/frontend/components/check_results/filters.py +++ b/vectordb_bench/frontend/components/check_results/filters.py @@ -1,14 +1,15 @@ from vectordb_bench.backend.cases import Case from vectordb_bench.frontend.components.check_results.data import getChartData -from vectordb_bench.frontend.components.check_results.expanderStyle import initSidebarExanderStyle -from vectordb_bench.frontend.config.dbCaseConfigs import CASE_NAME_ORDER -from vectordb_bench.frontend.config.styles import * +from vectordb_bench.frontend.components.check_results.expanderStyle import ( + initSidebarExanderStyle, +) +from vectordb_bench.frontend.config.styles import SIDEBAR_CONTROL_COLUMNS import streamlit as st from vectordb_bench.models import CaseResult, TestResult -def getshownData(results: list[TestResult], st): +def getshownData(st, results: list[TestResult], display_case_name_order: list[str]): # hide the nav st.markdown( "", @@ -17,16 +18,17 @@ def getshownData(results: list[TestResult], st): st.header("Filters") - shownResults = getshownResults(results, st) - showDBNames, showCaseNames = getShowDbsAndCases(shownResults, st) + shownResults = getshownResults(st, results) + showDBNames, showCaseNames = getShowDbsAndCases( + st, shownResults, display_case_name_order + ) - shownData, failedTasks = getChartData( - shownResults, showDBNames, showCaseNames) + shownData, failedTasks = getChartData(shownResults, showDBNames, showCaseNames) return shownData, failedTasks, showCaseNames -def getshownResults(results: list[TestResult], st) -> list[CaseResult]: +def getshownResults(st, results: list[TestResult]) -> list[CaseResult]: resultSelectOptions = [ result.task_label if result.task_label != result.run_id @@ -53,19 +55,24 @@ def getshownResults(results: list[TestResult], st) -> list[CaseResult]: return selectedResult -def getShowDbsAndCases(result: list[CaseResult], st) -> tuple[list[str], list[str]]: +def getShowDbsAndCases( + st, result: list[CaseResult], display_case_name_order +) -> tuple[list[str], list[str]]: initSidebarExanderStyle(st) allDbNames = list(set({res.task_config.db_name for res in result})) allDbNames.sort() allCases: list[Case] = [ res.task_config.case_config.case_id.case_cls( - res.task_config.case_config.custom_case) + res.task_config.case_config.custom_case + ) for res in result ] allCaseNameSet = set({case.name for case in allCases}) - allCaseNames = [case_name for case_name in CASE_NAME_ORDER if case_name in allCaseNameSet] + \ - [case_name for case_name in allCaseNameSet if case_name not in CASE_NAME_ORDER] - + allCaseNames = [ + case_name + for case_name in display_case_name_order + if case_name in allCaseNameSet + ] # DB Filter dbFilterContainer = st.container() showDBNames = filterView( @@ -120,8 +127,7 @@ def filterView(container, header, options, col, optionLables=None): ) if optionLables is None: optionLables = options - isActive = {option: st.session_state[selectAllState] - for option in optionLables} + isActive = {option: st.session_state[selectAllState] for option in optionLables} for i, option in enumerate(optionLables): isActive[option] = columns[i % col].checkbox( optionLables[i], diff --git a/vectordb_bench/frontend/components/tables/data.py b/vectordb_bench/frontend/components/tables/data.py index 96134c7ff..5d690fe84 100644 --- a/vectordb_bench/frontend/components/tables/data.py +++ b/vectordb_bench/frontend/components/tables/data.py @@ -1,5 +1,4 @@ from dataclasses import asdict -from vectordb_bench.backend.cases import CaseType from vectordb_bench.interface import benchMarkRunner from vectordb_bench.models import CaseResult, ResultLabel import pandas as pd @@ -14,7 +13,6 @@ def getNewResults(): for result in results: if result.label == ResultLabel.NORMAL: newResults.append(result) - df = pd.DataFrame(formatData(newResults)) return df @@ -26,8 +24,7 @@ def formatData(caseResults: list[CaseResult]): db = caseResult.task_config.db.value db_label = caseResult.task_config.db_config.db_label case_config = caseResult.task_config.case_config - db_case_config = caseResult.task_config.db_case_config - case = case_config.case_id.case_cls() + case = case_config.case filter_rate = case.filter_rate dataset = case.dataset.data.name metrics = asdict(caseResult.metrics) @@ -41,4 +38,4 @@ def formatData(caseResults: list[CaseResult]): **metrics, } ) - return data \ No newline at end of file + return data diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index ce8a3a4ae..188cb4ced 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -4,6 +4,7 @@ from vectordb_bench.backend.cases import CaseLabel, CaseType from vectordb_bench.backend.clients import DB from vectordb_bench.backend.clients.api import IndexType +from vectordb_bench.backend.dataset import DatasetWithSizeMap, DatasetWithSizeType from vectordb_bench.frontend.components.custom.getCustomConfig import get_custom_configs from vectordb_bench.models import CaseConfig, CaseConfigParamType @@ -33,21 +34,21 @@ class UICaseItem(BaseModel): def __init__( self, isLine: bool = False, - case_id: CaseType = None, - custom_case: dict = {}, - cases: list[CaseConfig] = [], + cases: list[CaseConfig] = None, label: str = "", description: str = "", caseLabel: CaseLabel = CaseLabel.Performance, ): if isLine is True: super().__init__(isLine=True) - elif case_id is not None and isinstance(case_id, CaseType): - c = case_id.case_cls(custom_case) + if cases is None: + cases = [] + elif len(cases) == 1: + c = cases[0].case super().__init__( label=c.name, description=c.description, - cases=[CaseConfig(case_id=case_id, custom_case=custom_case)], + cases=cases, caseLabel=c.label, ) else: @@ -71,7 +72,12 @@ def get_custom_case_items() -> list[UICaseItem]: custom_configs = get_custom_configs() return [ UICaseItem( - case_id=CaseType.PerformanceCustomDataset, custom_case=custom_config.dict() + cases=[ + CaseConfig( + case_id=CaseType.PerformanceCustomDataset, + custom_case=custom_config.dict(), + ) + ] ) for custom_config in custom_configs ] @@ -83,38 +89,73 @@ def get_custom_case_cluter() -> UICaseItemCluster: ) +def generate_normal_cases( + case_id: CaseType, custom_case: dict | None = None +) -> list[CaseConfig]: + return [CaseConfig(case_id=case_id, custom_case=custom_case)] + + +def generator_label_filter_cases( + dataset_with_size_type: DatasetWithSizeType +) -> list[CaseConfig]: + label_percentages = ( + dataset_with_size_type.get_manager().data.scalar_label_percentages + ) + return [ + CaseConfig( + case_id=CaseType.LabelFilterPerformanceCase, + custom_case=dict( + dataset_with_size_type=dataset_with_size_type, + label_percentage=label_percentage, + ), + ) + for label_percentage in label_percentages + ] + + UI_CASE_CLUSTERS: list[UICaseItemCluster] = [ UICaseItemCluster( label="Search Performance Test", uiCaseItems=[ - UICaseItem(case_id=CaseType.Performance768D100M), - UICaseItem(case_id=CaseType.Performance768D10M), - UICaseItem(case_id=CaseType.Performance768D1M), + UICaseItem(cases=generate_normal_cases(CaseType.Performance768D100M)), + UICaseItem(cases=generate_normal_cases(CaseType.Performance768D10M)), + UICaseItem(cases=generate_normal_cases(CaseType.Performance768D1M)), UICaseItem(isLine=True), - UICaseItem(case_id=CaseType.Performance1536D5M), - UICaseItem(case_id=CaseType.Performance1536D500K), - UICaseItem(case_id=CaseType.Performance1536D50K), + UICaseItem(cases=generate_normal_cases(CaseType.Performance1536D5M)), + UICaseItem(cases=generate_normal_cases(CaseType.Performance1536D500K)), + UICaseItem(cases=generate_normal_cases(CaseType.Performance1536D50K)), ], ), UICaseItemCluster( label="Filter Search Performance Test", uiCaseItems=[ - UICaseItem(case_id=CaseType.Performance768D10M1P), - UICaseItem(case_id=CaseType.Performance768D10M99P), - UICaseItem(case_id=CaseType.Performance768D1M1P), - UICaseItem(case_id=CaseType.Performance768D1M99P), + UICaseItem(cases=generate_normal_cases(CaseType.Performance768D10M1P)), + UICaseItem(cases=generate_normal_cases(CaseType.Performance768D10M99P)), + UICaseItem(cases=generate_normal_cases(CaseType.Performance768D1M1P)), + UICaseItem(cases=generate_normal_cases(CaseType.Performance768D1M99P)), UICaseItem(isLine=True), - UICaseItem(case_id=CaseType.Performance1536D5M1P), - UICaseItem(case_id=CaseType.Performance1536D5M99P), - UICaseItem(case_id=CaseType.Performance1536D500K1P), - UICaseItem(case_id=CaseType.Performance1536D500K99P), + UICaseItem(cases=generate_normal_cases(CaseType.Performance1536D5M1P)), + UICaseItem(cases=generate_normal_cases(CaseType.Performance1536D5M99P)), + UICaseItem(cases=generate_normal_cases(CaseType.Performance1536D500K1P)), + UICaseItem(cases=generate_normal_cases(CaseType.Performance1536D500K99P)), ], ), UICaseItemCluster( label="Capacity Test", uiCaseItems=[ - UICaseItem(case_id=CaseType.CapacityDim960), - UICaseItem(case_id=CaseType.CapacityDim128), + UICaseItem(cases=generate_normal_cases(CaseType.CapacityDim960)), + UICaseItem(cases=generate_normal_cases(CaseType.CapacityDim128)), + ], + ), + UICaseItemCluster( + label="Label-Filter Search Performance Test", + uiCaseItems=[ + UICaseItem( + label=f"Label-Filter Search Performance Test - {dataset_with_size_type.value}", + description=f"[Batch Cases], {dataset_with_size_type.get_manager().data.name}", + cases=generator_label_filter_cases(dataset_with_size_type), + ) + for dataset_with_size_type in DatasetWithSizeMap ], ), ] @@ -127,22 +168,11 @@ def get_custom_case_cluter() -> UICaseItemCluster: CaseType.Performance1536D5M, CaseType.Performance1536D500K, CaseType.Performance1536D50K, - CaseType.Performance768D10M1P, - CaseType.Performance768D1M1P, - CaseType.Performance1536D5M1P, - CaseType.Performance1536D500K1P, - CaseType.Performance768D10M99P, - CaseType.Performance768D1M99P, - CaseType.Performance1536D5M99P, - CaseType.Performance1536D500K99P, CaseType.CapacityDim960, CaseType.CapacityDim128, ] CASE_NAME_ORDER = [case.case_cls().name for case in DISPLAY_CASE_ORDER] -# CASE_LIST = [ -# item for item in CASE_LIST_WITH_DIVIDER if isinstance(item, CaseType)] - class InputType(IntEnum): Text = 20001 diff --git a/vectordb_bench/frontend/pages/filter.py b/vectordb_bench/frontend/pages/filter.py new file mode 100644 index 000000000..98fb94882 --- /dev/null +++ b/vectordb_bench/frontend/pages/filter.py @@ -0,0 +1,52 @@ +import streamlit as st +from vectordb_bench.frontend.components.check_results.footer import footer +from vectordb_bench.frontend.components.check_results.stPageConfig import ( + initResultsPageConfig, +) +from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon +from vectordb_bench.frontend.components.check_results.nav import ( + NavToQuriesPerDollar, + NavToRunTest, +) +from vectordb_bench.frontend.components.check_results.charts import drawCharts +from vectordb_bench.frontend.components.check_results.filters import getshownData +from vectordb_bench.frontend.components.get_results.saveAsImage import getResults + +# from vectordb_bench.frontend.config.styles import * +from vectordb_bench.interface import benchMarkRunner + + +def main(): + # set page config + initResultsPageConfig(st) + + # header + drawHeaderIcon(st) + + allResults = benchMarkRunner.get_results() + + st.title("Vector Database Benchmark") + st.caption( + "Note that all testing was completed in July 2023, except for the times already noted." + ) + + # results selector and filter + display_case_name_order = [] + resultSelectorContainer = st.sidebar.container() + shownData, failedTasks, showCaseNames = getshownData( + resultSelectorContainer, allResults, display_case_name_order + ) + + resultSelectorContainer.divider() + + # nav + navContainer = st.sidebar.container() + NavToRunTest(navContainer) + NavToQuriesPerDollar(navContainer) + + # footer + footer(st.container()) + + +if __name__ == "__main__": + main() diff --git a/vectordb_bench/frontend/vdb_benchmark.py b/vectordb_bench/frontend/vdb_benchmark.py index b859c68b8..69edf892c 100644 --- a/vectordb_bench/frontend/vdb_benchmark.py +++ b/vectordb_bench/frontend/vdb_benchmark.py @@ -1,12 +1,17 @@ import streamlit as st +from vectordb_bench.backend.cases import CaseType from vectordb_bench.frontend.components.check_results.footer import footer -from vectordb_bench.frontend.components.check_results.stPageConfig import initResultsPageConfig +from vectordb_bench.frontend.components.check_results.stPageConfig import ( + initResultsPageConfig, +) from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon -from vectordb_bench.frontend.components.check_results.nav import NavToQuriesPerDollar, NavToRunTest +from vectordb_bench.frontend.components.check_results.nav import ( + NavToQuriesPerDollar, + NavToRunTest, +) from vectordb_bench.frontend.components.check_results.charts import drawCharts from vectordb_bench.frontend.components.check_results.filters import getshownData from vectordb_bench.frontend.components.get_results.saveAsImage import getResults -from vectordb_bench.frontend.config.styles import * from vectordb_bench.interface import benchMarkRunner @@ -20,12 +25,27 @@ def main(): allResults = benchMarkRunner.get_results() st.title("Vector Database Benchmark") - st.caption("Note that all testing was completed in July 2023, except for the times already noted.") + st.caption( + "Note that all testing was completed in July 2023, except for the times already noted." + ) # results selector and filter resultSelectorContainer = st.sidebar.container() + display_case_name_order = [ + case_type.case_name() + for case_type in [ + CaseType.Performance768D100M, + CaseType.Performance768D10M, + CaseType.Performance768D1M, + CaseType.Performance1536D5M, + CaseType.Performance1536D500K, + CaseType.Performance1536D50K, + CaseType.CapacityDim960, + CaseType.CapacityDim128, + ] + ] shownData, failedTasks, showCaseNames = getshownData( - allResults, resultSelectorContainer + resultSelectorContainer, allResults, display_case_name_order ) resultSelectorContainer.divider() diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index 41be95b7c..c51ebc43f 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -11,7 +11,7 @@ DBConfig, DBCaseConfig, ) -from .backend.cases import CaseType +from .backend.cases import Case, CaseType from .base import BaseModel from . import config from .metric import Metric @@ -96,6 +96,14 @@ def k(self, value): def __hash__(self) -> int: return hash(self.json()) + @property + def case(self) -> Case: + return self.case_id.case_cls(self.custom_case) + + @property + def case_name(self) -> str: + return self.case.name + class TaskStage(StrEnum): """Enumerations of various stages of the task""" @@ -241,7 +249,7 @@ def display(self, dbs: list[DB] | None = None): key=lambda x: ( x.task_config.db.name, x.task_config.db_config.db_label, - x.task_config.case_config.case_id.name, + x.task_config.case_config.case_name, ), reverse=True, ) @@ -262,7 +270,7 @@ def append_return(x, y): + 3 ) max_case = max( - map(len, [f.task_config.case_config.case_id.name for f in filtered_results]) + map(len, [f.task_config.case_config.case_name for f in filtered_results]) ) max_load_dur = ( max(map(len, [str(f.metrics.load_duration) for f in filtered_results])) + 3 @@ -321,7 +329,7 @@ def append_return(x, y): % ( f.task_config.db.name, f.task_config.db_config.db_label, - f.task_config.case_config.case_id.name, + f.task_config.case_config.case_name, self.task_label, f.metrics.load_duration, f.metrics.qps, diff --git a/vectordb_bench/results/Milvus/result_20240722_2024072218_milvus.json b/vectordb_bench/results/Milvus/result_20240722_2024072218_milvus.json new file mode 100644 index 000000000..6be2446fa --- /dev/null +++ b/vectordb_bench/results/Milvus/result_20240722_2024072218_milvus.json @@ -0,0 +1,529 @@ +{ + "run_id": "bf9bc02236f4472f898d10d841d6141c", + "task_label": "2024072218", + "results": [ + { + "metrics": { + "max_load_count": 0, + "load_duration": 1122.0143, + "qps": 3453.8806, + "serial_latency_p99": 0.0028, + "recall": 0.9893, + "ndcg": 0.9912, + "conc_num_list": [ + 1, + 5 + ], + "conc_qps_list": [ + 625.2033, + 3453.8806 + ], + "conc_latency_p99_list": [ + 0.0011626282326877117, + 0.0010559258218854667 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.001 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1122.0143, + "qps": 3578.4471, + "serial_latency_p99": 0.0026, + "recall": 0.9386, + "ndcg": 0.947, + "conc_num_list": [ + 1, + 5 + ], + "conc_qps_list": [ + 631.0762, + 3578.4471 + ], + "conc_latency_p99_list": [ + 0.001154118150472641, + 0.0010039187181741 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.002 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1122.0143, + "qps": 3357.5076, + "serial_latency_p99": 0.0027, + "recall": 0.923, + "ndcg": 0.9333, + "conc_num_list": [ + 1, + 5 + ], + "conc_qps_list": [ + 682.1142, + 3357.5076 + ], + "conc_latency_p99_list": [ + 0.0011487580932676791, + 0.00110124426856637 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.005 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1122.0143, + "qps": 3211.4404, + "serial_latency_p99": 0.0029, + "recall": 0.9685, + "ndcg": 0.9737, + "conc_num_list": [ + 1, + 5 + ], + "conc_qps_list": [ + 639.1189, + 3211.4404 + ], + "conc_latency_p99_list": [ + 0.0012432025253772735, + 0.0011570992395281792 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.01 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1122.0143, + "qps": 3026.6245, + "serial_latency_p99": 0.0027, + "recall": 0.9555, + "ndcg": 0.963, + "conc_num_list": [ + 1, + 5 + ], + "conc_qps_list": [ + 556.0934, + 3026.6245 + ], + "conc_latency_p99_list": [ + 0.0013331254832446575, + 0.0012420137822628022 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.02 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1122.0143, + "qps": 2700.4768, + "serial_latency_p99": 0.0031, + "recall": 0.9422, + "ndcg": 0.9507, + "conc_num_list": [ + 1, + 5 + ], + "conc_qps_list": [ + 505.6855, + 2700.4768 + ], + "conc_latency_p99_list": [ + 0.0015245784275233745, + 0.0014426194280385972 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.05 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1122.0143, + "qps": 2346.9516, + "serial_latency_p99": 0.0031, + "recall": 0.9318, + "ndcg": 0.9422, + "conc_num_list": [ + 1, + 5 + ], + "conc_qps_list": [ + 446.5954, + 2346.9516 + ], + "conc_latency_p99_list": [ + 0.0018245578050613404, + 0.0017295872464776038 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.1 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1122.0143, + "qps": 1865.392, + "serial_latency_p99": 0.0043, + "recall": 0.9234, + "ndcg": 0.935, + "conc_num_list": [ + 1, + 5 + ], + "conc_qps_list": [ + 376.6724, + 1865.392 + ], + "conc_latency_p99_list": [ + 0.0023446583196520807, + 0.0022455231733620167 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.2 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1122.0143, + "qps": 1279.7205, + "serial_latency_p99": 0.0057, + "recall": 0.9456, + "ndcg": 0.9531, + "conc_num_list": [ + 1, + 5 + ], + "conc_qps_list": [ + 265.4133, + 1279.7205 + ], + "conc_latency_p99_list": [ + 0.0032637504577636717, + 0.003337749195098877 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json" +} \ No newline at end of file diff --git a/vectordb_bench/results/Milvus/result_20240723_2024072315_milvus.json b/vectordb_bench/results/Milvus/result_20240723_2024072315_milvus.json new file mode 100644 index 000000000..1dc2415d0 --- /dev/null +++ b/vectordb_bench/results/Milvus/result_20240723_2024072315_milvus.json @@ -0,0 +1,529 @@ +{ + "run_id": "c6e6512ff35c4ea1867f131d903549a8", + "task_label": "2024072315", + "results": [ + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 6629.3004, + "serial_latency_p99": 0.0031, + "recall": 0.99, + "ndcg": 0.9918, + "conc_num_list": [ + 5, + 10 + ], + "conc_qps_list": [ + 3525.1658, + 6629.3004 + ], + "conc_latency_p99_list": [ + 0.0010364896729588508, + 0.0010388683639466763 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.001 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 6799.3132, + "serial_latency_p99": 0.0029, + "recall": 0.938, + "ndcg": 0.9465, + "conc_num_list": [ + 5, + 10 + ], + "conc_qps_list": [ + 3531.2124, + 6799.3132 + ], + "conc_latency_p99_list": [ + 0.0010215921930968761, + 0.0010107415113598107 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.002 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 6283.4934, + "serial_latency_p99": 0.0028, + "recall": 0.9301, + "ndcg": 0.9399, + "conc_num_list": [ + 5, + 10 + ], + "conc_qps_list": [ + 3339.4558, + 6283.4934 + ], + "conc_latency_p99_list": [ + 0.001099529979377985, + 0.0010937995061278344 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.005 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 6067.9027, + "serial_latency_p99": 0.0029, + "recall": 0.9695, + "ndcg": 0.9749, + "conc_num_list": [ + 5, + 10 + ], + "conc_qps_list": [ + 3136.3638, + 6067.9027 + ], + "conc_latency_p99_list": [ + 0.0011826112452894449, + 0.001144677908346057 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.01 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 5755.0862, + "serial_latency_p99": 0.003, + "recall": 0.9584, + "ndcg": 0.9656, + "conc_num_list": [ + 5, + 10 + ], + "conc_qps_list": [ + 3002.1606, + 5755.0862 + ], + "conc_latency_p99_list": [ + 0.0012467310801148415, + 0.0012214824188500643 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.02 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 5021.8289, + "serial_latency_p99": 0.0031, + "recall": 0.9427, + "ndcg": 0.9518, + "conc_num_list": [ + 5, + 10 + ], + "conc_qps_list": [ + 2683.0249, + 5021.8289 + ], + "conc_latency_p99_list": [ + 0.0014433825548738242, + 0.001428482262417674 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.05 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 4270.1494, + "serial_latency_p99": 0.0033, + "recall": 0.9317, + "ndcg": 0.9421, + "conc_num_list": [ + 5, + 10 + ], + "conc_qps_list": [ + 2269.3802, + 4270.1494 + ], + "conc_latency_p99_list": [ + 0.0017319055356085301, + 0.0017313844017684459 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.1 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 3294.6054, + "serial_latency_p99": 0.0044, + "recall": 0.9232, + "ndcg": 0.935, + "conc_num_list": [ + 5, + 10 + ], + "conc_qps_list": [ + 1821.5784, + 3294.6054 + ], + "conc_latency_p99_list": [ + 0.0022613138016313312, + 0.002305802157521248 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.2 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 1635.2298, + "serial_latency_p99": 0.0058, + "recall": 0.9484, + "ndcg": 0.955, + "conc_num_list": [ + 5, + 10 + ], + "conc_qps_list": [ + 1287.4049, + 1635.2298 + ], + "conc_latency_p99_list": [ + 0.003252131423726678, + 0.003473675499856472 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json" +} \ No newline at end of file diff --git a/vectordb_bench/results/Milvus/result_20240723_2024072316_milvus.json b/vectordb_bench/results/Milvus/result_20240723_2024072316_milvus.json new file mode 100644 index 000000000..f765ede66 --- /dev/null +++ b/vectordb_bench/results/Milvus/result_20240723_2024072316_milvus.json @@ -0,0 +1,493 @@ +{ + "run_id": "261c1d3b673a403497460301e8e78c39", + "task_label": "2024072316", + "results": [ + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 6330.6515, + "serial_latency_p99": 0.0031, + "recall": 0.99, + "ndcg": 0.9918, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 6330.6515 + ], + "conc_latency_p99_list": [ + 0.0010543549612164497 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.001 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 6492.0424, + "serial_latency_p99": 0.003, + "recall": 0.938, + "ndcg": 0.9465, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 6492.0424 + ], + "conc_latency_p99_list": [ + 0.0010264522697776557 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.002 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 6097.6387, + "serial_latency_p99": 0.0026, + "recall": 0.9301, + "ndcg": 0.9399, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 6097.6387 + ], + "conc_latency_p99_list": [ + 0.001098549799621105 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.005 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 5735.1858, + "serial_latency_p99": 0.0029, + "recall": 0.9695, + "ndcg": 0.9749, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 5735.1858 + ], + "conc_latency_p99_list": [ + 0.0011668322145938874 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.01 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 5514.7877, + "serial_latency_p99": 0.003, + "recall": 0.9584, + "ndcg": 0.9656, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 5514.7877 + ], + "conc_latency_p99_list": [ + 0.0012358762323856354 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.02 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 4766.2055, + "serial_latency_p99": 0.0033, + "recall": 0.9427, + "ndcg": 0.9518, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 4766.2055 + ], + "conc_latency_p99_list": [ + 0.0014669299125671387 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.05 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 4262.1724, + "serial_latency_p99": 0.0036, + "recall": 0.9317, + "ndcg": 0.9421, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 4262.1724 + ], + "conc_latency_p99_list": [ + 0.0017046830385923385 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.1 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 3411.3493, + "serial_latency_p99": 0.0067, + "recall": 0.9232, + "ndcg": 0.935, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 3411.3493 + ], + "conc_latency_p99_list": [ + 0.002238515203818679 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.2 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 1728.1085, + "serial_latency_p99": 0.0095, + "recall": 0.9484, + "ndcg": 0.955, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 1728.1085 + ], + "conc_latency_p99_list": [ + 0.0033225916445255277 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json" +} \ No newline at end of file diff --git a/vectordb_bench/results/Milvus/result_20240723_shuffle_milvus.json b/vectordb_bench/results/Milvus/result_20240723_shuffle_milvus.json new file mode 100644 index 000000000..f078df648 --- /dev/null +++ b/vectordb_bench/results/Milvus/result_20240723_shuffle_milvus.json @@ -0,0 +1,529 @@ +{ + "run_id": "1cf492d97fd34a80a5a6ac62c254bde9", + "task_label": "shuffle", + "results": [ + { + "metrics": { + "max_load_count": 0, + "load_duration": 1120.518, + "qps": 6676.2282, + "serial_latency_p99": 0.0026, + "recall": 0.99, + "ndcg": 0.9918, + "conc_num_list": [ + 5, + 10 + ], + "conc_qps_list": [ + 3400.9913, + 6676.2282 + ], + "conc_latency_p99_list": [ + 0.0010667762741446494, + 0.00105709013864398 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.001 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1120.518, + "qps": 6603.0913, + "serial_latency_p99": 0.0026, + "recall": 0.938, + "ndcg": 0.9465, + "conc_num_list": [ + 5, + 10 + ], + "conc_qps_list": [ + 3516.5272, + 6603.0913 + ], + "conc_latency_p99_list": [ + 0.0010318856388330459, + 0.0010374930705875159 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.002 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1120.518, + "qps": 6335.2469, + "serial_latency_p99": 0.0026, + "recall": 0.9301, + "ndcg": 0.9399, + "conc_num_list": [ + 5, + 10 + ], + "conc_qps_list": [ + 3379.755, + 6335.2469 + ], + "conc_latency_p99_list": [ + 0.0010841774761676788, + 0.001089764142036438 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.005 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1120.518, + "qps": 5976.9228, + "serial_latency_p99": 0.0031, + "recall": 0.9695, + "ndcg": 0.9749, + "conc_num_list": [ + 5, + 10 + ], + "conc_qps_list": [ + 3183.8424, + 5976.9228 + ], + "conc_latency_p99_list": [ + 0.0011634252309799195, + 0.0011606734666973352 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.01 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1120.518, + "qps": 5712.5103, + "serial_latency_p99": 0.003, + "recall": 0.9584, + "ndcg": 0.9656, + "conc_num_list": [ + 5, + 10 + ], + "conc_qps_list": [ + 3051.625, + 5712.5103 + ], + "conc_latency_p99_list": [ + 0.0012276111483573915, + 0.001230158080905676 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.02 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1120.518, + "qps": 4998.7393, + "serial_latency_p99": 0.0034, + "recall": 0.9427, + "ndcg": 0.9518, + "conc_num_list": [ + 5, + 10 + ], + "conc_qps_list": [ + 2727.9194, + 4998.7393 + ], + "conc_latency_p99_list": [ + 0.0014320619486272335, + 0.0014417885534465313 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.05 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1120.518, + "qps": 4305.4345, + "serial_latency_p99": 0.0047, + "recall": 0.9317, + "ndcg": 0.9421, + "conc_num_list": [ + 5, + 10 + ], + "conc_qps_list": [ + 2322.894, + 4305.4345 + ], + "conc_latency_p99_list": [ + 0.0017353356435894965, + 0.00173754588291049 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.1 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1120.518, + "qps": 3476.5822, + "serial_latency_p99": 0.0053, + "recall": 0.9232, + "ndcg": 0.935, + "conc_num_list": [ + 5, + 10 + ], + "conc_qps_list": [ + 1864.2741, + 3476.5822 + ], + "conc_latency_p99_list": [ + 0.0022359615743160247, + 0.0022638415828347205 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.2 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1120.518, + "qps": 1721.3315, + "serial_latency_p99": 0.005, + "recall": 0.9484, + "ndcg": 0.955, + "conc_num_list": [ + 5, + 10 + ], + "conc_qps_list": [ + 1329.9625, + 1721.3315 + ], + "conc_latency_p99_list": [ + 0.0032103661209344865, + 0.003417107893154025 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json" +} \ No newline at end of file diff --git a/vectordb_bench/results/Milvus/result_20240723_test_milvus.json b/vectordb_bench/results/Milvus/result_20240723_test_milvus.json new file mode 100644 index 000000000..5be51ebdb --- /dev/null +++ b/vectordb_bench/results/Milvus/result_20240723_test_milvus.json @@ -0,0 +1,493 @@ +{ + "run_id": "b5db70bac69d48569b5df1eb69326ee3", + "task_label": "test", + "results": [ + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 6356.1596, + "serial_latency_p99": 0.0028, + "recall": 0.99, + "ndcg": 0.9918, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 6356.1596 + ], + "conc_latency_p99_list": [ + 0.0010720944307744504 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.001 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 6434.3303, + "serial_latency_p99": 0.0028, + "recall": 0.938, + "ndcg": 0.9465, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 6434.3303 + ], + "conc_latency_p99_list": [ + 0.001032950559258461 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.002 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 6110.2108, + "serial_latency_p99": 0.0029, + "recall": 0.9301, + "ndcg": 0.9399, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 6110.2108 + ], + "conc_latency_p99_list": [ + 0.001106204753369093 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.005 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 5816.4559, + "serial_latency_p99": 0.0028, + "recall": 0.9695, + "ndcg": 0.9749, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 5816.4559 + ], + "conc_latency_p99_list": [ + 0.001160949468612671 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.01 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 5481.0983, + "serial_latency_p99": 0.0031, + "recall": 0.9584, + "ndcg": 0.9656, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 5481.0983 + ], + "conc_latency_p99_list": [ + 0.0012512041836977004 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.02 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 4704.1764, + "serial_latency_p99": 0.0033, + "recall": 0.9427, + "ndcg": 0.9518, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 4704.1764 + ], + "conc_latency_p99_list": [ + 0.0014758364398032428 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.05 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 4225.0494, + "serial_latency_p99": 0.0039, + "recall": 0.9317, + "ndcg": 0.9421, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 4225.0494 + ], + "conc_latency_p99_list": [ + 0.001738075878471136 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.1 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 3326.3479, + "serial_latency_p99": 0.004, + "recall": 0.9232, + "ndcg": 0.935, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 3326.3479 + ], + "conc_latency_p99_list": [ + 0.002286043243482709 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.2 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 1686.7546, + "serial_latency_p99": 0.0049, + "recall": 0.9484, + "ndcg": 0.955, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 1686.7546 + ], + "conc_latency_p99_list": [ + 0.0034006748493760823 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "shuffle", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json" +} \ No newline at end of file diff --git a/vectordb_bench/results/Milvus/result_20240725_2024072418_milvus.json b/vectordb_bench/results/Milvus/result_20240725_2024072418_milvus.json new file mode 100644 index 000000000..44ac8b6e9 --- /dev/null +++ b/vectordb_bench/results/Milvus/result_20240725_2024072418_milvus.json @@ -0,0 +1,1015 @@ +{ + "run_id": "9a0813966f66424781eb59a47bbda0f5", + "task_label": "2024072418", + "results": [ + { + "metrics": { + "max_load_count": 0, + "load_duration": 11273.092, + "qps": 2874.737, + "serial_latency_p99": 0.0036, + "recall": 0.9925, + "ndcg": 0.9945, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 2874.737 + ], + "conc_latency_p99_list": [ + 0.001592563921958208 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "test", + "version": "", + "note": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Large OpenAI (1536dim, 5M)", + "label_percentage": 0.001 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 11273.092, + "qps": 5162.9449, + "serial_latency_p99": 0.0032, + "recall": 0.9799, + "ndcg": 0.9841, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 5162.9449 + ], + "conc_latency_p99_list": [ + 0.0013143851902335882 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "test", + "version": "", + "note": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Large OpenAI (1536dim, 5M)", + "label_percentage": 0.002 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 11273.092, + "qps": 4509.1693, + "serial_latency_p99": 0.0031, + "recall": 0.9732, + "ndcg": 0.9786, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 4509.1693 + ], + "conc_latency_p99_list": [ + 0.0015232897579669953 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "test", + "version": "", + "note": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Large OpenAI (1536dim, 5M)", + "label_percentage": 0.005 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 11273.092, + "qps": 3925.7059, + "serial_latency_p99": 0.0037, + "recall": 0.9682, + "ndcg": 0.9741, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 3925.7059 + ], + "conc_latency_p99_list": [ + 0.0017639627486467362 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "test", + "version": "", + "note": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Large OpenAI (1536dim, 5M)", + "label_percentage": 0.01 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 11273.092, + "qps": 3414.6608, + "serial_latency_p99": 0.0037, + "recall": 0.9633, + "ndcg": 0.97, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 3414.6608 + ], + "conc_latency_p99_list": [ + 0.00209084153175354 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "test", + "version": "", + "note": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Large OpenAI (1536dim, 5M)", + "label_percentage": 0.02 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 11273.092, + "qps": 1999.3002, + "serial_latency_p99": 0.0051, + "recall": 0.9751, + "ndcg": 0.9796, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 1999.3002 + ], + "conc_latency_p99_list": [ + 0.0027226969990879297 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "test", + "version": "", + "note": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Large OpenAI (1536dim, 5M)", + "label_percentage": 0.05 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 11273.092, + "qps": 1210.1864, + "serial_latency_p99": 0.0087, + "recall": 0.9832, + "ndcg": 0.9862, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 1210.1864 + ], + "conc_latency_p99_list": [ + 0.0031968073584139347 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "test", + "version": "", + "note": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Large OpenAI (1536dim, 5M)", + "label_percentage": 0.1 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 11273.092, + "qps": 631.0735, + "serial_latency_p99": 0.0168, + "recall": 0.9868, + "ndcg": 0.9892, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 631.0735 + ], + "conc_latency_p99_list": [ + 0.004965412712842226 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "test", + "version": "", + "note": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Large OpenAI (1536dim, 5M)", + "label_percentage": 0.2 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 11273.092, + "qps": 263.0213, + "serial_latency_p99": 0.0183, + "recall": 0.9903, + "ndcg": 0.9922, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 263.0213 + ], + "conc_latency_p99_list": [ + 0.011685187865793705 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "test", + "version": "", + "note": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Large OpenAI (1536dim, 5M)", + "label_percentage": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 12667.817, + "qps": 2847.4732, + "serial_latency_p99": 0.0037, + "recall": 0.9911, + "ndcg": 0.9925, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 2847.4732 + ], + "conc_latency_p99_list": [ + 0.0014339568197727203 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "test", + "version": "", + "note": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.001 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 12667.817, + "qps": 5316.1808, + "serial_latency_p99": 0.0031, + "recall": 0.9663, + "ndcg": 0.9723, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 5316.1808 + ], + "conc_latency_p99_list": [ + 0.001278534485027194 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "test", + "version": "", + "note": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.002 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 12667.817, + "qps": 4643.0748, + "serial_latency_p99": 0.0036, + "recall": 0.9486, + "ndcg": 0.9568, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 4643.0748 + ], + "conc_latency_p99_list": [ + 0.0014945681750774384 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "test", + "version": "", + "note": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.005 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 12667.817, + "qps": 4000.4843, + "serial_latency_p99": 0.0035, + "recall": 0.9324, + "ndcg": 0.9427, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 4000.4843 + ], + "conc_latency_p99_list": [ + 0.0018004775047302246 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "test", + "version": "", + "note": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.01 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 12667.817, + "qps": 3270.503, + "serial_latency_p99": 0.0042, + "recall": 0.9237, + "ndcg": 0.9343, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 3270.503 + ], + "conc_latency_p99_list": [ + 0.00232573615424335 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "test", + "version": "", + "note": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.02 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 12667.817, + "qps": 1613.3312, + "serial_latency_p99": 0.0053, + "recall": 0.9478, + "ndcg": 0.9535, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 1613.3312 + ], + "conc_latency_p99_list": [ + 0.0033007789179682733 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "test", + "version": "", + "note": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.05 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 12667.817, + "qps": 922.9687, + "serial_latency_p99": 0.0051, + "recall": 0.9567, + "ndcg": 0.9605, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 922.9687 + ], + "conc_latency_p99_list": [ + 0.00404267856515944 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "test", + "version": "", + "note": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.1 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 12667.817, + "qps": 506.543, + "serial_latency_p99": 0.008, + "recall": 0.9679, + "ndcg": 0.9696, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 506.543 + ], + "conc_latency_p99_list": [ + 0.0059432901553809645 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "test", + "version": "", + "note": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.2 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 12667.817, + "qps": 229.2432, + "serial_latency_p99": 0.0173, + "recall": 0.974, + "ndcg": 0.9744, + "conc_num_list": [ + 10 + ], + "conc_qps_list": [ + 229.2432 + ], + "conc_latency_p99_list": [ + 0.012065762715414166 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "test", + "version": "", + "note": "", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 102, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json" +} \ No newline at end of file diff --git a/vectordb_bench/results/getLeaderboardData.py b/vectordb_bench/results/getLeaderboardData.py index c6484514d..ea9d6a753 100644 --- a/vectordb_bench/results/getLeaderboardData.py +++ b/vectordb_bench/results/getLeaderboardData.py @@ -26,7 +26,7 @@ def main(): "db": d.task_config.db.value, "db_label": d.task_config.db_config.db_label, "db_name": d.task_config.db_name, - "case": d.task_config.case_config.case_id.case_name, + "case": d.task_config.case_config.case_name, "qps": d.metrics.qps, "latency": d.metrics.serial_latency_p99, "recall": d.metrics.recall,