diff --git a/neurips23/latitude-m4-metal-medium.md b/neurips23/latitude-m4-metal-medium.md index 874bd135..031e2a07 100644 --- a/neurips23/latitude-m4-metal-medium.md +++ b/neurips23/latitude-m4-metal-medium.md @@ -1,208 +1,213 @@ # Eval On AMD 3GHz/16-Core + 125GB RAM + NVMe SSD (Bare Metal) -## Table Of Contents - -- [Introduction](#introduction) -- [Results](#results) -- [Hardware Inventory](#hardware_inventory) -- [How To Reproduce](#how_to_reproduce) -- [Disclaimers And Credits](#disclaimers_and_credits) - -## Introduction - -The NeurIPS2023 Practical Vector Search Challenge evaluated participating algorithms on Azure and EC2 CPU-based hardware instances. - -In pursuit of expanding the evaluation criteria, we are also running on other generally available hardware configurations. - -Shown here are results run on the following hardware: -* AMD EPYC 9124 16-Core 3GHz processor -* 125GB RAM -* 440GB NVMe SSD -* Bare-metal "m4-metal-medium" instance provided by [Latitude](https://www.latitude.sh/) - -## Results - +
- - - + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + +
filtersparseoodfiltersparseood
rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K
2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K
3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
spmat
8
vamana
 20.0K8
faiss
 7.3K
sustech-whu
8
vamana
 20.0K
9
cufe
 6.3K
sustech-whu
9
puck
 19.0K9
cufe
 6.3K
spmat
9
puck
 19.0K
fdufilterdiskann
10
ngt
 11.9K
dhq
10
ngt
 11.9K
hwtl_sdu_anns_filter
11
epsearch
 7.7K
fdufilterdiskann
11
epsearch
 7.7K
dhq
12
diskann
 6.4K
hwtl_sdu_anns_filter
12
diskann
 6.4K
13
cufe
 5.4K13
cufe
 5.4K
puck-fizz
puck-fizz
+## Table Of Contents + +- [Introduction](#introduction) +- [Results](#results) +- [Hardware Inventory](#hardware_inventory) +- [How To Reproduce](#how_to_reproduce) +- [Disclaimers And Credits](#disclaimers_and_credits) + +## Introduction + +The NeurIPS2023 Practical Vector Search Challenge evaluated participating algorithms on Azure and EC2 CPU-based hardware instances. + +In pursuit of expanding the evaluation criteria, we are also running on other generally available hardware configurations. + +Shown here are results run on the following hardware: +* AMD EPYC 9124 16-Core 3GHz processor +* 125GB RAM +* 440GB NVMe SSD +* Bare-metal "m4-metal-medium" instance provided by [Latitude](https://www.latitude.sh/) + +## Results + +The calculated rankings are shown at the top. + Notes: -* Evaluations were run in late August 2024 -* In the table, each track algorithm links to the build and run commmand used (and errors if any). +* Evaluations were run in late August 2024. +* In each track, qualifying algorithms are ranked by largest *qps* where *recall/ap* >= 0.9. +* All participating algorithms are shown for each track, but only qualifying algorithms are ranked. +* Each track algorithm links to the build and run commmand used (or disqualifying errors, if any). * Pareto graphs for each track shown below. ### Track: Filter @@ -291,7 +296,7 @@ See the [latitude/commands](latitude/commands) directory for individual algorith To extract the data as CSV: ``` -sudo chmod ugo+rw -R ./results/ # recursively add read permissions to data files +sudo chmod ugo+rw -R ./results/ # recursively add read/write permissions to directories and files under the results directory. python data_export.py --recompute --output neurips23/latitude/data_export_m4-metal-medium.csv ``` diff --git a/neurips23/latitude/_latitude-m4-metal-medium.md b/neurips23/latitude/_latitude-m4-metal-medium.md index ab29e3ba..72e4167f 100644 --- a/neurips23/latitude/_latitude-m4-metal-medium.md +++ b/neurips23/latitude/_latitude-m4-metal-medium.md @@ -23,9 +23,13 @@ Shown here are results run on the following hardware: ## Results +The calculated rankings are shown at the top. + Notes: -* Evaluations were run in late August 2024 -* In the table, each track algorithm links to the build and run commmand used (and errors if any). +* Evaluations were run in late August 2024. +* In each track, qualifying algorithms are ranked by largest *qps* where *recall/ap* >= 0.9. +* All participating algorithms are shown for each track, but only qualifying algorithms are ranked. +* Each track algorithm links to the build and run commmand used (or disqualifying errors, if any). * Pareto graphs for each track shown below. ### Track: Filter @@ -114,7 +118,7 @@ See the [latitude/commands](latitude/commands) directory for individual algorith To extract the data as CSV: ``` -sudo chmod ugo+rw -R ./results/ # recursively add read permissions to data files +sudo chmod ugo+rw -R ./results/ # recursively add read/write permissions to directories and files under the results directory. python data_export.py --recompute --output neurips23/latitude/data_export_m4-metal-medium.csv ``` diff --git a/neurips23/latitude/analysis.ipynb b/neurips23/latitude/analysis.ipynb index a93b297f..524e519c 100644 --- a/neurips23/latitude/analysis.ipynb +++ b/neurips23/latitude/analysis.ipynb @@ -15,9 +15,6 @@ "# Relative path to competition directory\n", "COMPETITION_DIR = \"..\" \n", "\n", - "# Competition dataset names\n", - "DATASETS = []\n", - "\n", "# Competition recall/ap threshold\n", "RECALL_AP_THRESHOLD = 0.9\n", "\n", @@ -65,7 +62,7 @@ "source": [ "# read CSV\n", "\n", - "df = pd.read_csv( CSV )\n" + "df = pd.read_csv( CSV )" ] }, { @@ -268,7 +265,7 @@ " \n", " \n", " 9\n", - " dhq\n", + " fdufilterdiskann\n", " NaN\n", " NaN\n", " error\n", @@ -278,7 +275,7 @@ " \n", " \n", " 10\n", - " hwtl_sdu_anns_filter\n", + " dhq\n", " NaN\n", " NaN\n", " error\n", @@ -288,7 +285,7 @@ " \n", " \n", " 11\n", - " fdufilterdiskann\n", + " hwtl_sdu_anns_filter\n", " NaN\n", " NaN\n", " error\n", @@ -541,9 +538,9 @@ "6 faissplus 8.49 0.91 ok 7 filter \n", "7 faiss 7.33 0.99 ok 8 filter \n", "8 cufe 6.32 0.94 ok 9 filter \n", - "9 dhq NaN NaN error 10 filter \n", - "10 hwtl_sdu_anns_filter NaN NaN error 11 filter \n", - "11 fdufilterdiskann NaN NaN error 12 filter \n", + "9 fdufilterdiskann NaN NaN error 10 filter \n", + "10 dhq NaN NaN error 11 filter \n", + "11 hwtl_sdu_anns_filter NaN NaN error 12 filter \n", "0 pinecone-ood 76.87 0.91 ok 1 ood \n", "1 zilliz 73.51 0.91 ok 2 ood \n", "2 pyanns 55.50 0.92 ok 3 ood \n", @@ -976,7 +973,7 @@ " \n", " \n", " 9\n", - " dhq\n", + " fdufilterdiskann\n", " NaN\n", " NaN\n", " error\n", @@ -986,7 +983,7 @@ " \n", " \n", " 10\n", - " hwtl_sdu_anns_filter\n", + " dhq\n", " NaN\n", " NaN\n", " error\n", @@ -996,7 +993,7 @@ " \n", " \n", " 11\n", - " fdufilterdiskann\n", + " hwtl_sdu_anns_filter\n", " NaN\n", " NaN\n", " error\n", @@ -1042,9 +1039,9 @@ "6 faissplus 8.49 0.91 ok 7 filter \n", "7 faiss 7.33 0.99 ok 8 filter \n", "8 cufe 6.32 0.94 ok 9 filter \n", - "9 dhq NaN NaN error 10 filter \n", - "10 hwtl_sdu_anns_filter NaN NaN error 11 filter \n", - "11 fdufilterdiskann NaN NaN error 12 filter \n", + "9 fdufilterdiskann NaN NaN error 10 filter \n", + "10 dhq NaN NaN error 11 filter \n", + "11 hwtl_sdu_anns_filter NaN NaN error 12 filter \n", "\n", " dataset \n", "0 sparse-full \n", @@ -1109,7 +1106,7 @@ " columns=['track'], \n", " values=['algorithm','qps','status'],\n", " aggfunc='first')\n", - "#pivot_df\n" + "#pivot_df" ] }, { @@ -1174,25 +1171,25 @@ "text/html": [ "\n", - "\n", + "
\n", " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", @@ -1209,178 +1206,178 @@ " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
trackfiltersparseoodfiltersparseood
 rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
rank
11zilliz213.2850591zilliz34.7599761pinecone-ood76.865423
22pinecone146.7212422pyanns26.8585192zilliz73.509610
33puck62.2576953pinecone_smips12.0031823pyanns55.504352
44parlayivf55.0331984shnsw8.2472774scann32.256046
55wm_filter20.8748575nle2.9456265sustech-ood28.458263
66pyanns8.9880226cufe0.0849766mysteryann-dif27.946097
77faissplus8.4932647linscan0.0607567mysteryann26.560867
88faiss7.327855sustech-whunan8vamana19.965517
99cufe6.318462spmatnan9puck18.960224
10dhqnannannan10ngt11.921399
11hwtl_sdu_anns_filternannannan11epsearch7.663219
12fdufilterdiskannnannannan12diskann6.406821
13nannannannan13cufe5.404492
14nannannannanpuck-fizznan11zilliz213.2850591zilliz34.7599761pinecone-ood76.865423
22pinecone146.7212422pyanns26.8585192zilliz73.509610
33puck62.2576953pinecone_smips12.0031823pyanns55.504352
44parlayivf55.0331984shnsw8.2472774scann32.256046
55wm_filter20.8748575nle2.9456265sustech-ood28.458263
66pyanns8.9880226cufe0.0849766mysteryann-dif27.946097
77faissplus8.4932647linscan0.0607567mysteryann26.560867
88faiss7.327855sustech-whunan8vamana19.965517
99cufe6.318462spmatnan9puck18.960224
10fdufilterdiskannnannannan10ngt11.921399
11dhqnannannan11epsearch7.663219
12hwtl_sdu_anns_filternannannan12diskann6.406821
13nannannannan13cufe5.404492
14nannannannanpuck-fizznan
\n" ], "text/plain": [ - "" + "" ] }, "execution_count": 13, @@ -1407,188 +1404,188 @@ "text": [ "WARNING: Using error file path for sparse/sustech-whu\n", "WARNING: Using error file path for sparse/spmat\n", + "WARNING: Using error file path for filter/fdufilterdiskann\n", "WARNING: Using error file path for filter/dhq\n", "WARNING: Using error file path for filter/hwtl_sdu_anns_filter\n", - "WARNING: Using error file path for filter/fdufilterdiskann\n", "WARNING: Using error file path for ood/puck-fizz\n" ] }, { "data": { "text/html": [ - "\n", + "
\n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
filtersparseoodfiltersparseood
rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K
2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K
3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
sustech-whu
8
vamana
 20.0K
9
cufe
 6.3K
spmat
9
puck
 19.0K
dhq
10
ngt
 11.9K
hwtl_sdu_anns_filter
11
epsearch
 7.7K
fdufilterdiskann
12
diskann
 6.4K
13
cufe
 5.4K
puck-fizz
1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K
2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K
3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
sustech-whu
8
vamana
 20.0K
9
cufe
 6.3K
spmat
9
puck
 19.0K
fdufilterdiskann
10
ngt
 11.9K
dhq
11
epsearch
 7.7K
hwtl_sdu_anns_filter
12
diskann
 6.4K
13
cufe
 5.4K
puck-fizz
\n" @@ -1652,11 +1649,11 @@ " contents = md.read()\n", " \n", "# locate insertion point via regex\n", - "regexpr = \"(## Results\\n)\"\n", + "regexpr = \"(# Eval.*\\n)\"\n", "matches = re.search(regexpr, contents, re.M)\n", "\n", "# insert table html\n", - "new_contents = contents[0:matches.span(0)[1]] + new_html +\" \\n\" + contents[matches.span(0)[1]:]\n", + "new_contents = contents[0:matches.span(0)[1]] + \"\\n\" + new_html +\" \\n\" + contents[matches.span(0)[1]:]\n", "#print(new_contents[0:1000])\n", "\n", "# update the markdown\n",