Issue #211 - Fix RIS HTTP timeouts

dnas/dnas/config.py

@@ -158,6 +158,12 @@ class config:
     # Base dir to save MRT files to
     DL_DIR = os.path.join(DATA_DIR, "downloads/")
 
+    # Number of retires for downloading an MRT file
+    DL_RETIRES = 3
+
+    # Seconds between retires
+    DL_DELAY = 10
+
     # Temporary directory to split MRT files into
     SPLIT_DIR = "/tmp/"  # Set to None to disable MRT splitting
 

dnas/dnas/mrt_getter.py

@@ -1,6 +1,7 @@
 import datetime
 import logging
 import os
+from time import sleep
 from typing import Literal, Tuple, Union
 
 import requests
@@ -183,13 +184,30 @@ def download_file(
         logging.info(f"Downloading {url} to {filename}")
         try:
             """
-            The default Accept-Encoding is gzip, which causes the server to
-            respond with a Content-Length which is not the full file
-            size. Replace this so we can later compare the file size:
+            When pulling files from RIPE RIS, there are occasional timeouts.
+            Retry a few times with a pause between attempts because this seems
+            to almost always be a transient error.
             """
-            req = requests.get(
-                url, headers={"Accept-Encoding": "*"}, stream=True
-            )
+            retries = cfg.DL_RETIRES
+            while retries > 0:
+                try:
+                    """
+                    The default Accept-Encoding is gzip, which causes the server
+                    to respond with a Content-Length which is not the full file
+                    size. Replace this so we can later compare the file size:
+                    """
+                    req = requests.get(
+                        url, headers={"Accept-Encoding": "*"}, stream=True
+                    )
+                    retries = 0
+                except requests.exceptions.ReadTimeout as e:
+                    retries -= 1
+                    logging.info(
+                        f"Request timeout connecting to HTTP server: {e}\n"
+                        f"Remaining retires: {retries}\n"
+                        f"Waiting {cfg.DL_DELAY} seconds..."
+                    )
+                    sleep(cfg.DL_DELAY)
         except requests.exceptions.ConnectionError as e:
             logging.info(f"Couldn't connect to HTTP server: {e}")
             raise requests.exceptions.ConnectionError

dnas/dnas/mrt_stats.py

@@ -1258,7 +1258,7 @@ def from_json(self: "mrt_stats", json_str: str) -> None:
                 mrt_e = mrt_entry()
                 mrt_e.from_json(json_e)
                 self.bogon_origin_asns.append(mrt_e)
-        #else:
+        # else:
         #    self.bogon_origin_asns.append(mrt_entry())
         # Breaks report generations -> remove if stable after commenting out
 
@@ -1268,7 +1268,7 @@ def from_json(self: "mrt_stats", json_str: str) -> None:
                 mrt_e = mrt_entry()
                 mrt_e.from_json(json_e)
                 self.bogon_prefixes.append(mrt_e)
-        #else:
+        # else:
         #    self.bogon_prefixes.append(mrt_entry())
         # Breaks report generations -> remove if stable after commenting out
 
@@ -1278,7 +1278,7 @@ def from_json(self: "mrt_stats", json_str: str) -> None:
                 mrt_e = mrt_entry()
                 mrt_e.from_json(json_e)
                 self.highest_med_prefixes.append(mrt_e)
-        #else:
+        # else:
         #    self.highest_med_prefixes.append(mrt_entry())
         # Breaks report generations -> remove if stable after commenting out
 
@@ -1288,7 +1288,7 @@ def from_json(self: "mrt_stats", json_str: str) -> None:
                 mrt_e = mrt_entry()
                 mrt_e.from_json(json_e)
                 self.invalid_len.append(mrt_e)
-        #else:
+        # else:
         #    self.invalid_len.append(mrt_entry())
         # Breaks report generations -> remove if stable after commenting out
 
@@ -1316,7 +1316,7 @@ def from_json(self: "mrt_stats", json_str: str) -> None:
                 mrt_e = mrt_entry()
                 mrt_e.from_json(json_e)
                 self.most_bogon_asns.append(mrt_e)
-        #else:
+        # else:
         #    self.most_bogon_asns.append(mrt_entry())
         # Breaks report generations -> remove if stable after commenting out
 
@@ -1368,7 +1368,7 @@ def from_json(self: "mrt_stats", json_str: str) -> None:
                 mrt_e = mrt_entry()
                 mrt_e.from_json(json_e)
                 self.most_unknown_attrs.append(mrt_e)
-        #else:
+        # else:
         #    self.most_unknown_attrs.append(mrt_entry())
         # Breaks report generations -> remove if stable after commenting out
 
@@ -1378,7 +1378,7 @@ def from_json(self: "mrt_stats", json_str: str) -> None:
                 mrt_e = mrt_entry()
                 mrt_e.from_json(json_e)
                 self.most_unreg_origins.append(mrt_e)
-        #else:
+        # else:
         #    self.most_unreg_origins.append(mrt_entry())
         # Breaks report generations -> remove if stable after commenting out