From 9095fb63a1219a2bd8c2cbc22a940f23fb6c2fae Mon Sep 17 00:00:00 2001 From: George Williams Date: Thu, 29 Aug 2024 18:55:52 -0700 Subject: [PATCH 1/8] moved ranking to top of markdown --- neurips23/latitude/_latitude-m4-metal-medium.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/neurips23/latitude/_latitude-m4-metal-medium.md b/neurips23/latitude/_latitude-m4-metal-medium.md index ab29e3ba..f284345c 100644 --- a/neurips23/latitude/_latitude-m4-metal-medium.md +++ b/neurips23/latitude/_latitude-m4-metal-medium.md @@ -23,6 +23,8 @@ Shown here are results run on the following hardware: ## Results +The calculated rankings are shown at the top. + Notes: * Evaluations were run in late August 2024 * In the table, each track algorithm links to the build and run commmand used (and errors if any). From 5d4a762a85dbd4ab5fe6fb27132665e9f5160ac3 Mon Sep 17 00:00:00 2001 From: George Williams Date: Thu, 29 Aug 2024 18:56:13 -0700 Subject: [PATCH 2/8] added extra line breaks at top --- neurips23/latitude/analysis.ipynb | 732 +++++++++++++++--------------- 1 file changed, 368 insertions(+), 364 deletions(-) diff --git a/neurips23/latitude/analysis.ipynb b/neurips23/latitude/analysis.ipynb index a93b297f..7e645d72 100644 --- a/neurips23/latitude/analysis.ipynb +++ b/neurips23/latitude/analysis.ipynb @@ -268,7 +268,7 @@ " \n", " \n", " 9\n", - " dhq\n", + " hwtl_sdu_anns_filter\n", " NaN\n", " NaN\n", " error\n", @@ -278,7 +278,7 @@ " \n", " \n", " 10\n", - " hwtl_sdu_anns_filter\n", + " fdufilterdiskann\n", " NaN\n", " NaN\n", " error\n", @@ -288,7 +288,7 @@ " \n", " \n", " 11\n", - " fdufilterdiskann\n", + " dhq\n", " NaN\n", " NaN\n", " error\n", @@ -541,9 +541,9 @@ "6 faissplus 8.49 0.91 ok 7 filter \n", "7 faiss 7.33 0.99 ok 8 filter \n", "8 cufe 6.32 0.94 ok 9 filter \n", - "9 dhq NaN NaN error 10 filter \n", - "10 hwtl_sdu_anns_filter NaN NaN error 11 filter \n", - "11 fdufilterdiskann NaN NaN error 12 filter \n", + "9 hwtl_sdu_anns_filter NaN NaN error 10 filter \n", + "10 fdufilterdiskann NaN NaN error 11 filter \n", + "11 dhq NaN NaN error 12 filter \n", "0 pinecone-ood 76.87 0.91 ok 1 ood \n", "1 zilliz 73.51 0.91 ok 2 ood \n", "2 pyanns 55.50 0.92 ok 3 ood \n", @@ -976,7 +976,7 @@ " \n", " \n", " 9\n", - " dhq\n", + " hwtl_sdu_anns_filter\n", " NaN\n", " NaN\n", " error\n", @@ -986,7 +986,7 @@ " \n", " \n", " 10\n", - " hwtl_sdu_anns_filter\n", + " fdufilterdiskann\n", " NaN\n", " NaN\n", " error\n", @@ -996,7 +996,7 @@ " \n", " \n", " 11\n", - " fdufilterdiskann\n", + " dhq\n", " NaN\n", " NaN\n", " error\n", @@ -1042,9 +1042,9 @@ "6 faissplus 8.49 0.91 ok 7 filter \n", "7 faiss 7.33 0.99 ok 8 filter \n", "8 cufe 6.32 0.94 ok 9 filter \n", - "9 dhq NaN NaN error 10 filter \n", - "10 hwtl_sdu_anns_filter NaN NaN error 11 filter \n", - "11 fdufilterdiskann NaN NaN error 12 filter \n", + "9 hwtl_sdu_anns_filter NaN NaN error 10 filter \n", + "10 fdufilterdiskann NaN NaN error 11 filter \n", + "11 dhq NaN NaN error 12 filter \n", "\n", " dataset \n", "0 sparse-full \n", @@ -1174,25 +1174,25 @@ "text/html": [ "\n", - "\n", + "
\n", " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", @@ -1209,178 +1209,178 @@ " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
trackfiltersparseoodfiltersparseood
 rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
rank
11zilliz213.2850591zilliz34.7599761pinecone-ood76.865423
22pinecone146.7212422pyanns26.8585192zilliz73.509610
33puck62.2576953pinecone_smips12.0031823pyanns55.504352
44parlayivf55.0331984shnsw8.2472774scann32.256046
55wm_filter20.8748575nle2.9456265sustech-ood28.458263
66pyanns8.9880226cufe0.0849766mysteryann-dif27.946097
77faissplus8.4932647linscan0.0607567mysteryann26.560867
88faiss7.327855sustech-whunan8vamana19.965517
99cufe6.318462spmatnan9puck18.960224
10dhqnannannan10ngt11.921399
11hwtl_sdu_anns_filternannannan11epsearch7.663219
12fdufilterdiskannnannannan12diskann6.406821
13nannannannan13cufe5.404492
14nannannannanpuck-fizznan11zilliz213.2850591zilliz34.7599761pinecone-ood76.865423
22pinecone146.7212422pyanns26.8585192zilliz73.509610
33puck62.2576953pinecone_smips12.0031823pyanns55.504352
44parlayivf55.0331984shnsw8.2472774scann32.256046
55wm_filter20.8748575nle2.9456265sustech-ood28.458263
66pyanns8.9880226cufe0.0849766mysteryann-dif27.946097
77faissplus8.4932647linscan0.0607567mysteryann26.560867
88faiss7.327855sustech-whunan8vamana19.965517
99cufe6.318462spmatnan9puck18.960224
10hwtl_sdu_anns_filternannannan10ngt11.921399
11fdufilterdiskannnannannan11epsearch7.663219
12dhqnannannan12diskann6.406821
13nannannannan13cufe5.404492
14nannannannanpuck-fizznan
\n" ], "text/plain": [ - "" + "" ] }, "execution_count": 13, @@ -1407,188 +1407,188 @@ "text": [ "WARNING: Using error file path for sparse/sustech-whu\n", "WARNING: Using error file path for sparse/spmat\n", - "WARNING: Using error file path for filter/dhq\n", "WARNING: Using error file path for filter/hwtl_sdu_anns_filter\n", "WARNING: Using error file path for filter/fdufilterdiskann\n", + "WARNING: Using error file path for filter/dhq\n", "WARNING: Using error file path for ood/puck-fizz\n" ] }, { "data": { "text/html": [ - "\n", + "
\n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
filtersparseoodfiltersparseood
rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K
2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K
3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
sustech-whu
8
vamana
 20.0K
9
cufe
 6.3K
spmat
9
puck
 19.0K
dhq
10
ngt
 11.9K
hwtl_sdu_anns_filter
11
epsearch
 7.7K
fdufilterdiskann
12
diskann
 6.4K
13
cufe
 5.4K
puck-fizz
1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K
2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K
3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
sustech-whu
8
vamana
 20.0K
9
cufe
 6.3K
spmat
9
puck
 19.0K
hwtl_sdu_anns_filter
10
ngt
 11.9K
fdufilterdiskann
11
epsearch
 7.7K
dhq
12
diskann
 6.4K
13
cufe
 5.4K
puck-fizz
\n" @@ -1638,10 +1638,14 @@ "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wrote .././latitude-m4-metal-medium.md\n" + "ename": "TypeError", + "evalue": "bad operand type for unary +: 'str'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[15], line 11\u001b[0m\n\u001b[1;32m 8\u001b[0m matches \u001b[38;5;241m=\u001b[39m re\u001b[38;5;241m.\u001b[39msearch(regexpr, contents, re\u001b[38;5;241m.\u001b[39mM)\n\u001b[1;32m 10\u001b[0m \u001b[38;5;66;03m# insert table html\u001b[39;00m\n\u001b[0;32m---> 11\u001b[0m new_contents \u001b[38;5;241m=\u001b[39m contents[\u001b[38;5;241m0\u001b[39m:matches\u001b[38;5;241m.\u001b[39mspan(\u001b[38;5;241m0\u001b[39m)[\u001b[38;5;241m1\u001b[39m]] \u001b[38;5;241m+\u001b[39m \u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m \u001b[38;5;241m+\u001b[39m new_html \u001b[38;5;241m+\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m contents[matches\u001b[38;5;241m.\u001b[39mspan(\u001b[38;5;241m0\u001b[39m)[\u001b[38;5;241m1\u001b[39m]:]\n\u001b[1;32m 12\u001b[0m \u001b[38;5;66;03m#print(new_contents[0:1000])\u001b[39;00m\n\u001b[1;32m 13\u001b[0m \n\u001b[1;32m 14\u001b[0m \u001b[38;5;66;03m# update the markdown\u001b[39;00m\n\u001b[1;32m 15\u001b[0m fname \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin( \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m..\u001b[39m\u001b[38;5;124m\"\u001b[39m, MARKDOWN\u001b[38;5;241m.\u001b[39mreplace(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_\u001b[39m\u001b[38;5;124m\"\u001b[39m,\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m) )\n", + "\u001b[0;31mTypeError\u001b[0m: bad operand type for unary +: 'str'" ] } ], @@ -1652,11 +1656,11 @@ " contents = md.read()\n", " \n", "# locate insertion point via regex\n", - "regexpr = \"(## Results\\n)\"\n", + "regexpr = \"(# Eval.*\\n)\"\n", "matches = re.search(regexpr, contents, re.M)\n", "\n", "# insert table html\n", - "new_contents = contents[0:matches.span(0)[1]] + new_html +\" \\n\" + contents[matches.span(0)[1]:]\n", + "new_contents = contents[0:matches.span(0)[1]] + \"\\n\" + new_html +\" \\n\" + contents[matches.span(0)[1]:]\n", "#print(new_contents[0:1000])\n", "\n", "# update the markdown\n", From 595fd8f5f62f2f4d652bdf315f02d402d5f747a9 Mon Sep 17 00:00:00 2001 From: George Williams Date: Thu, 29 Aug 2024 18:57:36 -0700 Subject: [PATCH 3/8] re-render nb --- neurips23/latitude-m4-metal-medium.md | 323 +++++++++++++------------- 1 file changed, 163 insertions(+), 160 deletions(-) diff --git a/neurips23/latitude-m4-metal-medium.md b/neurips23/latitude-m4-metal-medium.md index 874bd135..a9b342a0 100644 --- a/neurips23/latitude-m4-metal-medium.md +++ b/neurips23/latitude-m4-metal-medium.md @@ -1,205 +1,208 @@ # Eval On AMD 3GHz/16-Core + 125GB RAM + NVMe SSD (Bare Metal) -## Table Of Contents - -- [Introduction](#introduction) -- [Results](#results) -- [Hardware Inventory](#hardware_inventory) -- [How To Reproduce](#how_to_reproduce) -- [Disclaimers And Credits](#disclaimers_and_credits) - -## Introduction - -The NeurIPS2023 Practical Vector Search Challenge evaluated participating algorithms on Azure and EC2 CPU-based hardware instances. - -In pursuit of expanding the evaluation criteria, we are also running on other generally available hardware configurations. - -Shown here are results run on the following hardware: -* AMD EPYC 9124 16-Core 3GHz processor -* 125GB RAM -* 440GB NVMe SSD -* Bare-metal "m4-metal-medium" instance provided by [Latitude](https://www.latitude.sh/) - -## Results - +
- - - + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + +
filtersparseoodfiltersparseood
rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K
2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K
3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
spmat
8
vamana
 20.0K8
faiss
 7.3K
sustech-whu
8
vamana
 20.0K
9
cufe
 6.3K
sustech-whu
9
puck
 19.0K9
cufe
 6.3K
spmat
9
puck
 19.0K
fdufilterdiskann
10
ngt
 11.9K
fdufilterdiskann
10
ngt
 11.9K
hwtl_sdu_anns_filter
11
epsearch
 7.7K
dhq
11
epsearch
 7.7K
dhq
12
diskann
 6.4K
hwtl_sdu_anns_filter
12
diskann
 6.4K
13
cufe
 5.4K13
cufe
 5.4K
puck-fizz
puck-fizz
+## Table Of Contents + +- [Introduction](#introduction) +- [Results](#results) +- [Hardware Inventory](#hardware_inventory) +- [How To Reproduce](#how_to_reproduce) +- [Disclaimers And Credits](#disclaimers_and_credits) + +## Introduction + +The NeurIPS2023 Practical Vector Search Challenge evaluated participating algorithms on Azure and EC2 CPU-based hardware instances. + +In pursuit of expanding the evaluation criteria, we are also running on other generally available hardware configurations. + +Shown here are results run on the following hardware: +* AMD EPYC 9124 16-Core 3GHz processor +* 125GB RAM +* 440GB NVMe SSD +* Bare-metal "m4-metal-medium" instance provided by [Latitude](https://www.latitude.sh/) + +## Results + +The calculated rankings are shown at the top. + Notes: * Evaluations were run in late August 2024 * In the table, each track algorithm links to the build and run commmand used (and errors if any). From b042555b9881999dd20e1826b70978924027521b Mon Sep 17 00:00:00 2001 From: George Williams Date: Thu, 29 Aug 2024 19:01:01 -0700 Subject: [PATCH 4/8] add 'qualifying' note --- neurips23/latitude-m4-metal-medium.md | 279 +++---- .../latitude/_latitude-m4-metal-medium.md | 1 + neurips23/latitude/analysis.ipynb | 728 +++++++++--------- 3 files changed, 503 insertions(+), 505 deletions(-) diff --git a/neurips23/latitude-m4-metal-medium.md b/neurips23/latitude-m4-metal-medium.md index a9b342a0..e3ba04dc 100644 --- a/neurips23/latitude-m4-metal-medium.md +++ b/neurips23/latitude-m4-metal-medium.md @@ -1,179 +1,179 @@ # Eval On AMD 3GHz/16-Core + 125GB RAM + NVMe SSD (Bare Metal) - +
- - - + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + +
filtersparseoodfiltersparseood
rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K
2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K
3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
sustech-whu
8
vamana
 20.0K8
faiss
 7.3K
sustech-whu
8
vamana
 20.0K
9
cufe
 6.3K
spmat
9
puck
 19.0K9
cufe
 6.3K
spmat
9
puck
 19.0K
fdufilterdiskann
10
ngt
 11.9K
hwtl_sdu_anns_filter
10
ngt
 11.9K
dhq
11
epsearch
 7.7K
fdufilterdiskann
11
epsearch
 7.7K
hwtl_sdu_anns_filter
12
diskann
 6.4K
dhq
12
diskann
 6.4K
13
cufe
 5.4K13
cufe
 5.4K
puck-fizz
puck-fizz
@@ -205,6 +205,7 @@ The calculated rankings are shown at the top. Notes: * Evaluations were run in late August 2024 +# All participating algorithms are shown, but only qualifying algorithms are ranked. * In the table, each track algorithm links to the build and run commmand used (and errors if any). * Pareto graphs for each track shown below. diff --git a/neurips23/latitude/_latitude-m4-metal-medium.md b/neurips23/latitude/_latitude-m4-metal-medium.md index f284345c..9f04f9b3 100644 --- a/neurips23/latitude/_latitude-m4-metal-medium.md +++ b/neurips23/latitude/_latitude-m4-metal-medium.md @@ -27,6 +27,7 @@ The calculated rankings are shown at the top. Notes: * Evaluations were run in late August 2024 +# All participating algorithms are shown, but only qualifying algorithms are ranked. * In the table, each track algorithm links to the build and run commmand used (and errors if any). * Pareto graphs for each track shown below. diff --git a/neurips23/latitude/analysis.ipynb b/neurips23/latitude/analysis.ipynb index 7e645d72..4f35edd3 100644 --- a/neurips23/latitude/analysis.ipynb +++ b/neurips23/latitude/analysis.ipynb @@ -268,7 +268,7 @@ " \n", " \n", " 9\n", - " hwtl_sdu_anns_filter\n", + " fdufilterdiskann\n", " NaN\n", " NaN\n", " error\n", @@ -278,7 +278,7 @@ " \n", " \n", " 10\n", - " fdufilterdiskann\n", + " dhq\n", " NaN\n", " NaN\n", " error\n", @@ -288,7 +288,7 @@ " \n", " \n", " 11\n", - " dhq\n", + " hwtl_sdu_anns_filter\n", " NaN\n", " NaN\n", " error\n", @@ -541,9 +541,9 @@ "6 faissplus 8.49 0.91 ok 7 filter \n", "7 faiss 7.33 0.99 ok 8 filter \n", "8 cufe 6.32 0.94 ok 9 filter \n", - "9 hwtl_sdu_anns_filter NaN NaN error 10 filter \n", - "10 fdufilterdiskann NaN NaN error 11 filter \n", - "11 dhq NaN NaN error 12 filter \n", + "9 fdufilterdiskann NaN NaN error 10 filter \n", + "10 dhq NaN NaN error 11 filter \n", + "11 hwtl_sdu_anns_filter NaN NaN error 12 filter \n", "0 pinecone-ood 76.87 0.91 ok 1 ood \n", "1 zilliz 73.51 0.91 ok 2 ood \n", "2 pyanns 55.50 0.92 ok 3 ood \n", @@ -976,7 +976,7 @@ " \n", " \n", " 9\n", - " hwtl_sdu_anns_filter\n", + " fdufilterdiskann\n", " NaN\n", " NaN\n", " error\n", @@ -986,7 +986,7 @@ " \n", " \n", " 10\n", - " fdufilterdiskann\n", + " dhq\n", " NaN\n", " NaN\n", " error\n", @@ -996,7 +996,7 @@ " \n", " \n", " 11\n", - " dhq\n", + " hwtl_sdu_anns_filter\n", " NaN\n", " NaN\n", " error\n", @@ -1042,9 +1042,9 @@ "6 faissplus 8.49 0.91 ok 7 filter \n", "7 faiss 7.33 0.99 ok 8 filter \n", "8 cufe 6.32 0.94 ok 9 filter \n", - "9 hwtl_sdu_anns_filter NaN NaN error 10 filter \n", - "10 fdufilterdiskann NaN NaN error 11 filter \n", - "11 dhq NaN NaN error 12 filter \n", + "9 fdufilterdiskann NaN NaN error 10 filter \n", + "10 dhq NaN NaN error 11 filter \n", + "11 hwtl_sdu_anns_filter NaN NaN error 12 filter \n", "\n", " dataset \n", "0 sparse-full \n", @@ -1174,25 +1174,25 @@ "text/html": [ "\n", - "\n", + "
\n", " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", @@ -1209,178 +1209,178 @@ " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
trackfiltersparseoodfiltersparseood
 rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
rank
11zilliz213.2850591zilliz34.7599761pinecone-ood76.865423
22pinecone146.7212422pyanns26.8585192zilliz73.509610
33puck62.2576953pinecone_smips12.0031823pyanns55.504352
44parlayivf55.0331984shnsw8.2472774scann32.256046
55wm_filter20.8748575nle2.9456265sustech-ood28.458263
66pyanns8.9880226cufe0.0849766mysteryann-dif27.946097
77faissplus8.4932647linscan0.0607567mysteryann26.560867
88faiss7.327855sustech-whunan8vamana19.965517
99cufe6.318462spmatnan9puck18.960224
10hwtl_sdu_anns_filternannannan10ngt11.921399
11fdufilterdiskannnannannan11epsearch7.663219
12dhqnannannan12diskann6.406821
13nannannannan13cufe5.404492
14nannannannanpuck-fizznan11zilliz213.2850591zilliz34.7599761pinecone-ood76.865423
22pinecone146.7212422pyanns26.8585192zilliz73.509610
33puck62.2576953pinecone_smips12.0031823pyanns55.504352
44parlayivf55.0331984shnsw8.2472774scann32.256046
55wm_filter20.8748575nle2.9456265sustech-ood28.458263
66pyanns8.9880226cufe0.0849766mysteryann-dif27.946097
77faissplus8.4932647linscan0.0607567mysteryann26.560867
88faiss7.327855sustech-whunan8vamana19.965517
99cufe6.318462spmatnan9puck18.960224
10fdufilterdiskannnannannan10ngt11.921399
11dhqnannannan11epsearch7.663219
12hwtl_sdu_anns_filternannannan12diskann6.406821
13nannannannan13cufe5.404492
14nannannannanpuck-fizznan
\n" ], "text/plain": [ - "" + "" ] }, "execution_count": 13, @@ -1407,188 +1407,188 @@ "text": [ "WARNING: Using error file path for sparse/sustech-whu\n", "WARNING: Using error file path for sparse/spmat\n", - "WARNING: Using error file path for filter/hwtl_sdu_anns_filter\n", "WARNING: Using error file path for filter/fdufilterdiskann\n", "WARNING: Using error file path for filter/dhq\n", + "WARNING: Using error file path for filter/hwtl_sdu_anns_filter\n", "WARNING: Using error file path for ood/puck-fizz\n" ] }, { "data": { "text/html": [ - "\n", + "
\n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
filtersparseoodfiltersparseood
rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K
2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K
3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
sustech-whu
8
vamana
 20.0K
9
cufe
 6.3K
spmat
9
puck
 19.0K
hwtl_sdu_anns_filter
10
ngt
 11.9K
fdufilterdiskann
11
epsearch
 7.7K
dhq
12
diskann
 6.4K
13
cufe
 5.4K
puck-fizz
1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K
2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K
3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
sustech-whu
8
vamana
 20.0K
9
cufe
 6.3K
spmat
9
puck
 19.0K
fdufilterdiskann
10
ngt
 11.9K
dhq
11
epsearch
 7.7K
hwtl_sdu_anns_filter
12
diskann
 6.4K
13
cufe
 5.4K
puck-fizz
\n" @@ -1638,14 +1638,10 @@ "metadata": {}, "outputs": [ { - "ename": "TypeError", - "evalue": "bad operand type for unary +: 'str'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[15], line 11\u001b[0m\n\u001b[1;32m 8\u001b[0m matches \u001b[38;5;241m=\u001b[39m re\u001b[38;5;241m.\u001b[39msearch(regexpr, contents, re\u001b[38;5;241m.\u001b[39mM)\n\u001b[1;32m 10\u001b[0m \u001b[38;5;66;03m# insert table html\u001b[39;00m\n\u001b[0;32m---> 11\u001b[0m new_contents \u001b[38;5;241m=\u001b[39m contents[\u001b[38;5;241m0\u001b[39m:matches\u001b[38;5;241m.\u001b[39mspan(\u001b[38;5;241m0\u001b[39m)[\u001b[38;5;241m1\u001b[39m]] \u001b[38;5;241m+\u001b[39m \u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m \u001b[38;5;241m+\u001b[39m new_html \u001b[38;5;241m+\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m contents[matches\u001b[38;5;241m.\u001b[39mspan(\u001b[38;5;241m0\u001b[39m)[\u001b[38;5;241m1\u001b[39m]:]\n\u001b[1;32m 12\u001b[0m \u001b[38;5;66;03m#print(new_contents[0:1000])\u001b[39;00m\n\u001b[1;32m 13\u001b[0m \n\u001b[1;32m 14\u001b[0m \u001b[38;5;66;03m# update the markdown\u001b[39;00m\n\u001b[1;32m 15\u001b[0m fname \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin( \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m..\u001b[39m\u001b[38;5;124m\"\u001b[39m, MARKDOWN\u001b[38;5;241m.\u001b[39mreplace(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_\u001b[39m\u001b[38;5;124m\"\u001b[39m,\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m) )\n", - "\u001b[0;31mTypeError\u001b[0m: bad operand type for unary +: 'str'" + "name": "stdout", + "output_type": "stream", + "text": [ + "Wrote .././latitude-m4-metal-medium.md\n" ] } ], From 44521a8bd64ed7f91a7a054d69e60d9acf384f41 Mon Sep 17 00:00:00 2001 From: George Williams Date: Thu, 29 Aug 2024 19:04:50 -0700 Subject: [PATCH 5/8] fix language and re-render nb --- neurips23/latitude-m4-metal-medium.md | 285 +++++++++--------- .../latitude/_latitude-m4-metal-medium.md | 7 +- 2 files changed, 147 insertions(+), 145 deletions(-) diff --git a/neurips23/latitude-m4-metal-medium.md b/neurips23/latitude-m4-metal-medium.md index e3ba04dc..e9cfb260 100644 --- a/neurips23/latitude-m4-metal-medium.md +++ b/neurips23/latitude-m4-metal-medium.md @@ -1,179 +1,179 @@ # Eval On AMD 3GHz/16-Core + 125GB RAM + NVMe SSD (Bare Metal) - +
- - - + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + +
filtersparseoodfiltersparseood
rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K
2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K
3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
sustech-whu
8
vamana
 20.0K8
faiss
 7.3K
spmat
8
vamana
 20.0K
9
cufe
 6.3K
spmat
9
puck
 19.0K9
cufe
 6.3K
sustech-whu
9
puck
 19.0K
hwtl_sdu_anns_filter
10
ngt
 11.9K
dhq
10
ngt
 11.9K
fdufilterdiskann
11
epsearch
 7.7K
hwtl_sdu_anns_filter
11
epsearch
 7.7K
dhq
12
diskann
 6.4K
fdufilterdiskann
12
diskann
 6.4K
13
cufe
 5.4K13
cufe
 5.4K
puck-fizz
puck-fizz
@@ -204,9 +204,10 @@ Shown here are results run on the following hardware: The calculated rankings are shown at the top. Notes: -* Evaluations were run in late August 2024 -# All participating algorithms are shown, but only qualifying algorithms are ranked. -* In the table, each track algorithm links to the build and run commmand used (and errors if any). +* Evaluations were run in late August 2024/Q +* In each track, qualifying algorithms are ranked by largest QPS where recall/ap >= 0.9. +* All participating algorithms are shown for each track, but only qualifying algorithms are ranked. +* Each track algorithm links to the build and run commmand used (or disqualifying errors, if any). * Pareto graphs for each track shown below. ### Track: Filter diff --git a/neurips23/latitude/_latitude-m4-metal-medium.md b/neurips23/latitude/_latitude-m4-metal-medium.md index 9f04f9b3..52f3af33 100644 --- a/neurips23/latitude/_latitude-m4-metal-medium.md +++ b/neurips23/latitude/_latitude-m4-metal-medium.md @@ -26,9 +26,10 @@ Shown here are results run on the following hardware: The calculated rankings are shown at the top. Notes: -* Evaluations were run in late August 2024 -# All participating algorithms are shown, but only qualifying algorithms are ranked. -* In the table, each track algorithm links to the build and run commmand used (and errors if any). +* Evaluations were run in late August 2024/Q +* In each track, qualifying algorithms are ranked by largest QPS where recall/ap >= 0.9. +* All participating algorithms are shown for each track, but only qualifying algorithms are ranked. +* Each track algorithm links to the build and run commmand used (or disqualifying errors, if any). * Pareto graphs for each track shown below. ### Track: Filter From 2e4d15f6d5d2dc91360a72efe46681ef78905ee4 Mon Sep 17 00:00:00 2001 From: George Williams Date: Thu, 29 Aug 2024 19:06:04 -0700 Subject: [PATCH 6/8] fix typo --- neurips23/latitude-m4-metal-medium.md | 280 +++++++++--------- .../latitude/_latitude-m4-metal-medium.md | 2 +- 2 files changed, 141 insertions(+), 141 deletions(-) diff --git a/neurips23/latitude-m4-metal-medium.md b/neurips23/latitude-m4-metal-medium.md index e9cfb260..2750be60 100644 --- a/neurips23/latitude-m4-metal-medium.md +++ b/neurips23/latitude-m4-metal-medium.md @@ -1,179 +1,179 @@ # Eval On AMD 3GHz/16-Core + 125GB RAM + NVMe SSD (Bare Metal) - +
- - - + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + +
filtersparseoodfiltersparseood
rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K
2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K
3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
spmat
8
vamana
 20.0K8
faiss
 7.3K
spmat
8
vamana
 20.0K
9
cufe
 6.3K
sustech-whu
9
puck
 19.0K9
cufe
 6.3K
sustech-whu
9
puck
 19.0K
dhq
10
ngt
 11.9K
fdufilterdiskann
10
ngt
 11.9K
hwtl_sdu_anns_filter
11
epsearch
 7.7K
dhq
11
epsearch
 7.7K
fdufilterdiskann
12
diskann
 6.4K
hwtl_sdu_anns_filter
12
diskann
 6.4K
13
cufe
 5.4K13
cufe
 5.4K
puck-fizz
puck-fizz
@@ -204,7 +204,7 @@ Shown here are results run on the following hardware: The calculated rankings are shown at the top. Notes: -* Evaluations were run in late August 2024/Q +* Evaluations were run in late August 2024. * In each track, qualifying algorithms are ranked by largest QPS where recall/ap >= 0.9. * All participating algorithms are shown for each track, but only qualifying algorithms are ranked. * Each track algorithm links to the build and run commmand used (or disqualifying errors, if any). diff --git a/neurips23/latitude/_latitude-m4-metal-medium.md b/neurips23/latitude/_latitude-m4-metal-medium.md index 52f3af33..0cdd744d 100644 --- a/neurips23/latitude/_latitude-m4-metal-medium.md +++ b/neurips23/latitude/_latitude-m4-metal-medium.md @@ -26,7 +26,7 @@ Shown here are results run on the following hardware: The calculated rankings are shown at the top. Notes: -* Evaluations were run in late August 2024/Q +* Evaluations were run in late August 2024. * In each track, qualifying algorithms are ranked by largest QPS where recall/ap >= 0.9. * All participating algorithms are shown for each track, but only qualifying algorithms are ranked. * Each track algorithm links to the build and run commmand used (or disqualifying errors, if any). From 8549d8d1d4e65c8a6fe8b22a70b1df59cd9bed38 Mon Sep 17 00:00:00 2001 From: George Williams Date: Thu, 29 Aug 2024 19:11:08 -0700 Subject: [PATCH 7/8] fix language --- neurips23/latitude-m4-metal-medium.md | 280 +++++++++--------- .../latitude/_latitude-m4-metal-medium.md | 2 +- 2 files changed, 141 insertions(+), 141 deletions(-) diff --git a/neurips23/latitude-m4-metal-medium.md b/neurips23/latitude-m4-metal-medium.md index 2750be60..fbcface9 100644 --- a/neurips23/latitude-m4-metal-medium.md +++ b/neurips23/latitude-m4-metal-medium.md @@ -1,179 +1,179 @@ # Eval On AMD 3GHz/16-Core + 125GB RAM + NVMe SSD (Bare Metal) - +
- - - + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + +
filtersparseoodfiltersparseood
rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K
2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K
3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
spmat
8
vamana
 20.0K8
faiss
 7.3K
spmat
8
vamana
 20.0K
9
cufe
 6.3K
sustech-whu
9
puck
 19.0K9
cufe
 6.3K
sustech-whu
9
puck
 19.0K
fdufilterdiskann
10
ngt
 11.9K
fdufilterdiskann
10
ngt
 11.9K
dhq
11
epsearch
 7.7K
dhq
11
epsearch
 7.7K
hwtl_sdu_anns_filter
12
diskann
 6.4K
hwtl_sdu_anns_filter
12
diskann
 6.4K
13
cufe
 5.4K13
cufe
 5.4K
puck-fizz
puck-fizz
@@ -296,7 +296,7 @@ See the [latitude/commands](latitude/commands) directory for individual algorith To extract the data as CSV: ``` -sudo chmod ugo+rw -R ./results/ # recursively add read permissions to data files +sudo chmod ugo+rw -R ./results/ # recursively add read/write permissions to directories and files under the results directory. python data_export.py --recompute --output neurips23/latitude/data_export_m4-metal-medium.csv ``` diff --git a/neurips23/latitude/_latitude-m4-metal-medium.md b/neurips23/latitude/_latitude-m4-metal-medium.md index 0cdd744d..8f23cbe0 100644 --- a/neurips23/latitude/_latitude-m4-metal-medium.md +++ b/neurips23/latitude/_latitude-m4-metal-medium.md @@ -118,7 +118,7 @@ See the [latitude/commands](latitude/commands) directory for individual algorith To extract the data as CSV: ``` -sudo chmod ugo+rw -R ./results/ # recursively add read permissions to data files +sudo chmod ugo+rw -R ./results/ # recursively add read/write permissions to directories and files under the results directory. python data_export.py --recompute --output neurips23/latitude/data_export_m4-metal-medium.csv ``` From 227cf9a2ff78ae37a2f4c1695040b6aee427add6 Mon Sep 17 00:00:00 2001 From: George Williams Date: Fri, 30 Aug 2024 08:32:57 -0700 Subject: [PATCH 8/8] fix language;re-render nb --- neurips23/latitude-m4-metal-medium.md | 280 +++++++++--------- .../latitude/_latitude-m4-metal-medium.md | 2 +- neurips23/latitude/analysis.ipynb | 7 +- 3 files changed, 143 insertions(+), 146 deletions(-) diff --git a/neurips23/latitude-m4-metal-medium.md b/neurips23/latitude-m4-metal-medium.md index fbcface9..031e2a07 100644 --- a/neurips23/latitude-m4-metal-medium.md +++ b/neurips23/latitude-m4-metal-medium.md @@ -1,179 +1,179 @@ # Eval On AMD 3GHz/16-Core + 125GB RAM + NVMe SSD (Bare Metal) - +
- - - + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + +
filtersparseoodfiltersparseood
rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K
2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K
3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
spmat
8
vamana
 20.0K8
faiss
 7.3K
sustech-whu
8
vamana
 20.0K
9
cufe
 6.3K
sustech-whu
9
puck
 19.0K9
cufe
 6.3K
spmat
9
puck
 19.0K
fdufilterdiskann
10
ngt
 11.9K
dhq
10
ngt
 11.9K
dhq
11
epsearch
 7.7K
fdufilterdiskann
11
epsearch
 7.7K
hwtl_sdu_anns_filter
12
diskann
 6.4K
hwtl_sdu_anns_filter
12
diskann
 6.4K
13
cufe
 5.4K13
cufe
 5.4K
puck-fizz
puck-fizz
@@ -205,7 +205,7 @@ The calculated rankings are shown at the top. Notes: * Evaluations were run in late August 2024. -* In each track, qualifying algorithms are ranked by largest QPS where recall/ap >= 0.9. +* In each track, qualifying algorithms are ranked by largest *qps* where *recall/ap* >= 0.9. * All participating algorithms are shown for each track, but only qualifying algorithms are ranked. * Each track algorithm links to the build and run commmand used (or disqualifying errors, if any). * Pareto graphs for each track shown below. diff --git a/neurips23/latitude/_latitude-m4-metal-medium.md b/neurips23/latitude/_latitude-m4-metal-medium.md index 8f23cbe0..72e4167f 100644 --- a/neurips23/latitude/_latitude-m4-metal-medium.md +++ b/neurips23/latitude/_latitude-m4-metal-medium.md @@ -27,7 +27,7 @@ The calculated rankings are shown at the top. Notes: * Evaluations were run in late August 2024. -* In each track, qualifying algorithms are ranked by largest QPS where recall/ap >= 0.9. +* In each track, qualifying algorithms are ranked by largest *qps* where *recall/ap* >= 0.9. * All participating algorithms are shown for each track, but only qualifying algorithms are ranked. * Each track algorithm links to the build and run commmand used (or disqualifying errors, if any). * Pareto graphs for each track shown below. diff --git a/neurips23/latitude/analysis.ipynb b/neurips23/latitude/analysis.ipynb index 4f35edd3..524e519c 100644 --- a/neurips23/latitude/analysis.ipynb +++ b/neurips23/latitude/analysis.ipynb @@ -15,9 +15,6 @@ "# Relative path to competition directory\n", "COMPETITION_DIR = \"..\" \n", "\n", - "# Competition dataset names\n", - "DATASETS = []\n", - "\n", "# Competition recall/ap threshold\n", "RECALL_AP_THRESHOLD = 0.9\n", "\n", @@ -65,7 +62,7 @@ "source": [ "# read CSV\n", "\n", - "df = pd.read_csv( CSV )\n" + "df = pd.read_csv( CSV )" ] }, { @@ -1109,7 +1106,7 @@ " columns=['track'], \n", " values=['algorithm','qps','status'],\n", " aggfunc='first')\n", - "#pivot_df\n" + "#pivot_df" ] }, {