Move Azure hosted data from anon container access to static website (#…

…183) * moved data to azure static website * moved data to azure static website * switches links to static websites
harsha-simhadri · Nov 9, 2023 · 3066793 · 3066793
1 parent 435d45b
commit 3066793
Show file tree

Hide file tree

Showing 6 changed files with 34 additions and 34 deletions.
diff --git a/algos-2021.yaml b/algos-2021.yaml
@@ -474,7 +474,7 @@ deep-1B:
         base:
           args: |
               [{"R":100, "L":100, "B":50, "M":110,
-                "url": "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/diskann-T2-baseline-indices/deep-1B/R100_L100_B50_M110"
+                "url": "https://comp21storage.z5.web.core.windows.net/comp21/diskann-T2-baseline-indices/deep-1B/R100_L100_B50_M110"
                 }]
           query-args: |
               [{"Ls":30, "BW":4, "T":16},
@@ -522,7 +522,7 @@ msspacev-1B:
         base:
           args: |
               [{"R":100, "L":100, "B":47, "M":100,
-                "url": "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/diskann-T2-baseline-indices/msspacev-1B/R100_L100_B47_M100"
+                "url": "https://comp21storage.z5.web.core.windows.net/comp21/diskann-T2-baseline-indices/msspacev-1B/R100_L100_B47_M100"
                 }]
           query-args: |
               [{"Ls":60, "BW":4, "T":16, "HE":3},
@@ -582,7 +582,7 @@ msspacev-1B:
         base:
           args: |
               [{"R":100, "L":100, "B":47, "M":100,
-                "url": "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/diskann-T2-baseline-indices/msspacev-1B/R100_L100_B47_M100"
+                "url": "https://comp21storage.z5.web.core.windows.net/comp21/diskann-T2-baseline-indices/msspacev-1B/R100_L100_B47_M100"
                 }]
           query-args: |
               [{"Ls":40, "BW":4, "T":16},
@@ -654,7 +654,7 @@ msturing-1B:
         base:
           args: |
               [{"R":100, "L":100, "B":50, "M":80,
-                "url": "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/diskann-T2-baseline-indices/msturing-1B/R100_L100_B50_M80"
+                "url": "https://comp21storage.z5.web.core.windows.net/comp21/diskann-T2-baseline-indices/msturing-1B/R100_L100_B50_M80"
                 }]
           query-args: |
               [{"Ls":30, "BW":4, "T":16},
@@ -758,7 +758,7 @@ msturing-1B:
         base:
           args: |
               [{"R":100, "L":100, "B":50, "M":80,
-                "url": "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/diskann-T2-baseline-indices/msturing-1B/R100_L100_B50_M80"
+                "url": "https://comp21storage.z5.web.core.windows.net/comp21/diskann-T2-baseline-indices/msturing-1B/R100_L100_B50_M80"
                 }]
           query-args: |
               [{"Ls":30, "BW":4, "T":16},
@@ -806,7 +806,7 @@ bigann-1B:
         base:
           args: |
               [{"R":100, "L":100, "B":50, "M":80,
-                "url": "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/diskann-T2-baseline-indices/bigann-1B/R100_L100_B50_M80"
+                "url": "https://comp21storage.z5.web.core.windows.net/comp21/diskann-T2-baseline-indices/bigann-1B/R100_L100_B50_M80"
                 }]
           query-args: |
               [{"Ls":30, "BW":4, "T":16},
@@ -938,7 +938,7 @@ bigann-1B:
         base:
           args: |
               [{"R":100, "L":100, "B":50, "M":80,
-                "url": "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/diskann-T2-baseline-indices/bigann-1B/R100_L100_B50_M80"
+                "url": "https://comp21storage.z5.web.core.windows.net/comp21/diskann-T2-baseline-indices/bigann-1B/R100_L100_B50_M80"
                 }]
           query-args: |
               [{"Ls":30, "BW":4, "T":16},
@@ -961,7 +961,7 @@ ssnpp-1B:
         base:
           args: |
               [{"R":100, "L":100, "B":60, "M":100, "C":500000, "CM":2,
-                "url": "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/diskann-T2-baseline-indices/fbssnpp-1B/R100_L100_B60_M100"
+                "url": "https://comp21storage.z5.web.core.windows.net/comp21/diskann-T2-baseline-indices/fbssnpp-1B/R100_L100_B60_M100"
                 }]
           query-args: |
               [{"Lmin":9, "Lmax":10000, "BW":4, "T":16},
@@ -1023,7 +1023,7 @@ ssnpp-1B:
         base:
           args: |
               [{"R":100, "L":100, "B":60, "M":100, "C":500000, "CM":2,
-                "url": "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/diskann-T2-baseline-indices/fbssnpp-1B/R100_L100_B60_M100"
+                "url": "https://comp21storage.z5.web.core.windows.net/comp21/diskann-T2-baseline-indices/fbssnpp-1B/R100_L100_B60_M100"
                 }]
           query-args: |
               [{"Lmin":8, "Lmax":10000, "BW":4, "T":16},
@@ -1129,7 +1129,7 @@ text2image-1B:
         base:
           args: |
               [{"R":100, "L":100, "B":60, "M":115, "PQ":200, "C":500000, "CM":2,
-                "url": "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/diskann-T2-baseline-indices/text2image-1B/R100_L100_B60_M115_PQ200"
+                "url": "https://comp21storage.z5.web.core.windows.net/comp21/diskann-T2-baseline-indices/text2image-1B/R100_L100_B60_M115_PQ200"
                 }]
           query-args: |
               [{"Ls":10, "BW":10, "T":16},

diff --git a/benchmark/datasets.py b/benchmark/datasets.py
@@ -314,7 +314,7 @@ def __init__(self, nb_M=1000):
             subset_url + "GT_10M/bigann-10M" if self.nb_M == 10 else
             None
         )
-        # self.gt_fn = "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/bigann/public_query_gt100.bin" if self.nb == 10**9 else None
+        # self.gt_fn = "https://comp21storage.z5.web.core.windows.net/comp21/bigann/public_query_gt100.bin" if self.nb == 10**9 else None
         self.base_url = "https://dl.fbaipublicfiles.com/billion-scale-ann-benchmarks/bigann/"
         self.basedir = os.path.join(BASEDIR, "bigann")
 
@@ -345,8 +345,8 @@ def __init__(self, nb_M=1000):
         self.basedir = os.path.join(BASEDIR, "deep1b")
 
         self.private_nq = 30000
-        self.private_qs_url = "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/deep1b/query.heldout.30K.fbin"
-        self.private_gt_url = "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/deep1b/gt100-heldout.30K.fbin"
+        self.private_qs_url = "https://comp21storage.z5.web.core.windows.net/comp21/deep1b/query.heldout.30K.fbin"
+        self.private_gt_url = "https://comp21storage.z5.web.core.windows.net/comp21/deep1b/gt100-heldout.30K.fbin"
 
         self.private_nq_large = 1000000
         self.private_qs_large_url = "https://storage.yandexcloud.net/yr-secret-share/ann-datasets-5ac0659e27/DEEP/query.private.1M.fbin"
@@ -406,16 +406,16 @@ def __init__(self, nb_M=1000):
             "msturing-gt-1M" if self.nb_M == 1 else
             None
         )
-        self.base_url = "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/MSFT-TURING-ANNS/"
+        self.base_url = "https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/"
         self.basedir = os.path.join(BASEDIR, "MSTuringANNS")
 
         self.private_nq = 10000
-        self.private_qs_url = "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/MSFT-TURING-ANNS/testQuery10K.fbin"
-        self.private_gt_url = "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/MSFT-TURING-ANNS/gt100-private10K-queries.bin"
+        self.private_qs_url = "https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/testQuery10K.fbin"
+        self.private_gt_url = "https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/gt100-private10K-queries.bin"
 
         self.private_nq_large = 99605
-        self.private_qs_large_url = "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/MSFT-TURING-ANNS/testQuery99605.fbin"
-        self.private_gt_large_url = "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/MSFT-TURING-ANNS/gt100-private99605-queries.bin"
+        self.private_qs_large_url = "https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/testQuery99605.fbin"
+        self.private_gt_large_url = "https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/gt100-private99605-queries.bin"
 
     def distance(self):
         return "euclidean"
@@ -430,7 +430,7 @@ def __init__(self):
         self.qs_fn = "testQuery10K.fbin"
         self.gt_fn = "clu_msturing10M_gt100"
 
-        self.base_url = "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/clustered_data/msturing-10M-clustered/"
+        self.base_url = "https://comp21storage.z5.web.core.windows.net/comp23/clustered_data/msturing-10M-clustered/"
         self.basedir = os.path.join(BASEDIR, "MSTuring-10M-clustered")
 
         self.private_gt_url = None
@@ -452,7 +452,7 @@ def __init__(self):
         self.qs_fn = "testQuery10K.fbin"
         self.gt_fn = "clu_msturing30M_gt100"
 
-        self.base_url = "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/clustered_data/msturing-30M-clustered/"
+        self.base_url = "https://comp21storage.z5.web.core.windows.net/comp23/clustered_data/msturing-30M-clustered/"
         self.basedir = os.path.join(BASEDIR, "MSTuring-30M-clustered")
 
         self.private_gt_url = None
@@ -480,12 +480,12 @@ def __init__(self, nb_M=1000):
             "msspacev-gt-1M" if self.nb_M == 1 else
             None
         )
-        self.base_url = "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/spacev1b/"
+        self.base_url = "https://comp21storage.z5.web.core.windows.net/comp21/spacev1b/"
         self.basedir = os.path.join(BASEDIR, "MSSPACEV1B")
 
         self.private_nq = 30000
-        self.private_qs_url = "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/spacev1b/private_query_30k.bin"
-        self.private_gt_url = "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/spacev1b/gt100_private_query_30k.bin"
+        self.private_qs_url = "https://comp21storage.z5.web.core.windows.net/comp21/spacev1b/private_query_30k.bin"
+        self.private_gt_url = "https://comp21storage.z5.web.core.windows.net/comp21/spacev1b/gt100_private_query_30k.bin"
 
     def distance(self):
         return "euclidean"
@@ -500,7 +500,7 @@ def __init__(self, basedir="random-clustered"):
         self.qs_fn = f"queries_1000_20.fbin"
         self.gt_fn = f"clu_random_gt100"
 
-        self.base_url="https://comp21storage.blob.core.windows.net/publiccontainer/comp23/clustered_data/random-xs-clustered/"
+        self.base_url="https://comp21storage.z5.web.core.windows.net/comp23/clustered_data/random-xs-clustered/"
 
         self.basedir = os.path.join(BASEDIR, f"{basedir}{self.nb}")
         if not os.path.exists(self.basedir):

diff --git a/neurips23/streaming/clustered_runbook.yaml b/neurips23/streaming/clustered_runbook.yaml
@@ -192,7 +192,7 @@ random-xs-clustered:
     end: 10000
   64:
     operation: "search"
-  gt_url: "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/clustered_data/random-xs-clustered/clustered_runboook.yaml"
+  gt_url: "https://comp21storage.z5.web.core.windows.net/comp23/clustered_data/random-xs-clustered/clustered_runboook.yaml"
 msturing-10M-clustered:
   max_pts: 10000000
   # On Azure D8lds v5, runtime: 14 minutes
@@ -389,4 +389,4 @@ msturing-10M-clustered:
     end: 10000000
   64:
     operation: "search"
-  gt_url: "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/clustered_data/msturing-10M-clustered/clustered_runboook.yaml"
+  gt_url: "https://comp21storage.z5.web.core.windows.net/comp23/clustered_data/msturing-10M-clustered/clustered_runboook.yaml"
diff --git a/neurips23/streaming/delete_runbook.yaml b/neurips23/streaming/delete_runbook.yaml
@@ -240,7 +240,7 @@ random-xs-clustered:
     end: 8000
   76:
     operation: "search"
-  gt_url: "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/clustered_data/random-xs-clustered/delete_runbook.yaml"
+  gt_url: "https://comp21storage.z5.web.core.windows.net/comp23/clustered_data/random-xs-clustered/delete_runbook.yaml"
 msturing-10M-clustered:
   max_pts: 6000000
   # On Azure D8lds v5, runtime: ~23 minutes 
@@ -479,4 +479,4 @@ msturing-10M-clustered:
     end: 9200000
   75:
     operation: "search"
-  gt_url: "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/clustered_data/msturing-10M-clustered/delete_runbook.yaml"
+  gt_url: "https://comp21storage.z5.web.core.windows.net/comp23/clustered_data/msturing-10M-clustered/delete_runbook.yaml"
diff --git a/neurips23/streaming/final_runbook.yaml b/neurips23/streaming/final_runbook.yaml
@@ -3840,4 +3840,4 @@ msturing-30M-clustered:
     end: 29991665
   1280:
     operation: 'search'
-  gt_url: "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/clustered_data/msturing-30M-clustered/final_runbook.yaml"
+  gt_url: "https://comp21storage.z5.web.core.windows.net/comp23/clustered_data/msturing-30M-clustered/final_runbook.yaml"
diff --git a/neurips23/streaming/simple_runbook.yaml b/neurips23/streaming/simple_runbook.yaml
@@ -18,7 +18,7 @@ random-xs:
     end: 5000
   6:
     operation: "search"
-  gt_url: "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/str_gt/random10000/10000/simple_runbook.yaml"
+  gt_url: "https://comp21storage.z5.web.core.windows.net/comp23/str_gt/random10000/10000/simple_runbook.yaml"
 msturing-10M:
   max_pts: 10000000
   # On Azure D8lds v5 with "R":50, "L":50, "insert_threads":16, "consolidate_threads":16
@@ -41,7 +41,7 @@ msturing-10M:
     end: 5000000
   6:
     operation: "search"
-  gt_url: "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/str_gt/MSTuringANNS/10000000/simple_runbook.yaml"
+  gt_url: "https://comp21storage.z5.web.core.windows.net/comp23/str_gt/MSTuringANNS/10000000/simple_runbook.yaml"
 msturing-1M:
   max_pts: 1000000
   # On Azure D8lds v5 with "R":50, "L":50, "insert_threads":16, "consolidate_threads":16
@@ -65,7 +65,7 @@ msturing-1M:
     end: 500000
   6:
     operation: "search"
-  gt_url: "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/str_gt/MSTuringANNS/1000000/simple_runbook.yaml"
+  gt_url: "https://comp21storage.z5.web.core.windows.net/comp23/str_gt/MSTuringANNS/1000000/simple_runbook.yaml"
 msspacev-10M:
   max_pts: 10000000
   1: 
@@ -86,7 +86,7 @@ msspacev-10M:
     end: 5000000
   6:
     operation: "search"
-  gt_url: "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/str_gt/MSSPACEV1B/10000000/simple_runbook.yaml"
+  gt_url: "https://comp21storage.z5.web.core.windows.net/comp23/str_gt/MSSPACEV1B/10000000/simple_runbook.yaml"
 msspacev-1M:
   max_pts: 1000000
   1: 
@@ -107,4 +107,4 @@ msspacev-1M:
     end: 500000
   6:
     operation: "search"
-  gt_url: "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/str_gt/MSSPACEV1B/1000000/simple_runbook.yaml"
+  gt_url: "https://comp21storage.z5.web.core.windows.net/comp23/str_gt/MSSPACEV1B/1000000/simple_runbook.yaml"