Add timeout control and retry logic for http requests obtaining sandb… (

#1269)

* Add timeout control and retry logic for http requests obtaining sandbox files.

* Fix the way that I was sending the timeout to httpx. It's now being passed for both the connect and read timeouts.

* mypy

---------

Co-authored-by: Eric Patey <>
Co-authored-by: jjallaire <jj.allaire@gmail.com>

src/inspect_ai/_eval/task/sandbox.py

@@ -5,11 +5,20 @@
 from typing import AsyncGenerator, Callable, NamedTuple, cast
 
 import httpx
+from tenacity import (
+    retry,
+    retry_if_exception,
+    stop_after_attempt,
+    stop_after_delay,
+    wait_exponential_jitter,
+)
 
 from inspect_ai._eval.task.task import Task
 from inspect_ai._eval.task.util import task_run_dir
+from inspect_ai._util.constants import DEFAULT_MAX_RETRIES, DEFAULT_TIMEOUT
 from inspect_ai._util.file import file, filesystem
 from inspect_ai._util.registry import registry_unqualified_name
+from inspect_ai._util.retry import httpx_should_retry, log_retry_attempt
 from inspect_ai._util.url import data_uri_to_base64, is_data_uri, is_http_url
 from inspect_ai.dataset import Sample
 from inspect_ai.util._concurrency import concurrency
@@ -115,8 +124,7 @@ async def read_sandboxenv_file(contents: str) -> bytes:
         contents_base64 = data_uri_to_base64(contents)
         file_bytes = base64.b64decode(contents_base64)
     elif is_http_url(contents):
-        client = httpx.AsyncClient()
-        file_bytes = (await client.get(contents, follow_redirects=True)).content
+        file_bytes = await _retrying_httpx_get(contents)
     else:
         # try to read as a file (if it doesn't exist or has a path not cool w/
         # the filesystem then we fall back to contents)
@@ -172,3 +180,28 @@ def resolve_sandbox(
         return sample.sandbox
     else:
         return None
+
+
+async def _retrying_httpx_get(
+    url: str,
+    client: httpx.AsyncClient = httpx.AsyncClient(),
+    timeout: int = 30,  # per-attempt timeout
+    max_retries: int = DEFAULT_MAX_RETRIES,
+    total_timeout: int = DEFAULT_TIMEOUT,  #  timeout for the whole retry loop. not for an individual attempt
+) -> bytes:
+    @retry(
+        wait=wait_exponential_jitter(),
+        stop=(stop_after_attempt(max_retries) | stop_after_delay(total_timeout)),
+        retry=retry_if_exception(httpx_should_retry),
+        before_sleep=log_retry_attempt(url),
+    )
+    async def do_get() -> bytes:
+        response = await client.get(
+            url=url,
+            follow_redirects=True,
+            timeout=(timeout, timeout, timeout, timeout),
+        )
+        response.raise_for_status()
+        return response.content
+
+    return await do_get()