🔖 Release 3.4.5 (#77)

**Fixed**
- Thread-safety issue when leveraging a single multiplexed connection
across multiple threads.
- Apparently consumed content when allow_redirect is set to True when
accessing a lazy response that follow redirects.

**Changed**
- urllib3.future lower bound constraint has been raised to version
2.5.900 in order to leverage the advanced multiplexing scheduler. This
upgrade come with a noticeable performance bump in a complex multiplexed
context.

**Added**
- ``Session`` constructor now accepts both ``pool_connections`` and
``pool_maxsize`` parameters to scale your pools of connections at will.

HISTORY.md

@@ -1,6 +1,20 @@
 Release History
 ===============
 
+3.4.5 (2024-02-02)
+------------------
+
+**Fixed**
+- Thread-safety issue when leveraging a single multiplexed connection across multiple threads.
+- Apparently consumed content when allow_redirect is set to True when accessing a lazy response that follow redirects.
+
+**Changed**
+- urllib3.future lower bound constraint has been raised to version 2.5.900 in order to leverage the advanced multiplexing scheduler.
+  This upgrade come with a noticeable performance bump.
+
+**Added**
+- ``Session`` constructor now accepts both ``pool_connections`` and ``pool_maxsize`` parameters to scale your pools of connections at will.
+
 3.4.4 (2024-01-18)
 ------------------
 

docs/user/quickstart.rst

@@ -694,8 +694,6 @@ Any ``Response`` returned by get, post, put, etc... will be a lazy instance of `
 
 The possible algorithms are actually nearly limitless, and you may arrange/write you own scheduling technics!
 
-.. warning:: Beware that all in-flight (unresolved) lazy responses are lost immediately after closing the ``Session``. Trying to access unresolved and lost responses will result in ``MultiplexingError`` exception being raised.
-
 Session Gather
 --------------
 
@@ -872,6 +870,23 @@ Here is a basic example of how you would do it::
 
 .. warning:: Accessing a lazy ``AsyncResponse`` without a call to ``s.gather()`` will raise a warning.
 
+Scale your Session / Pool
+-------------------------
+
+By default, Niquests allow, concurrently 10 hosts, and 10 connections per host.
+You can at your own discretion increase or decrease the values.
+
+To do so, you are invited to set the following parameters within a Session constructor:
+
+``Session(pool_connections=10, pool_maxsize=10)``
+
+- **pool_connections** means the number of host target (or pool of connections if you prefer).
+- **pool_maxsize** means the maximum of concurrent connexion per host target/pool.
+
+.. warning:: Due to the multiplexed aspect of both HTTP/2, and HTTP/3 you can issue, usually, more than 200 requests per connection without ever needing to create another one.
+
+.. note:: This setting is most useful for multi-threading application.
+
 DNS Resolution
 --------------
 

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "niquests"
-description = "Python HTTP for Humans."
+description = "Niquests is a simple, yet elegant, HTTP library. It is a drop-in replacement for Requests, which is under feature freeze."
 readme = "README.md"
 license-files = { paths = ["LICENSE"] }
 license = "Apache-2.0"
@@ -41,7 +41,7 @@ dynamic = ["version"]
 dependencies = [
     "charset_normalizer>=2,<4",
     "idna>=2.5,<4",
-    "urllib3.future>=2.4.904,<3",
+    "urllib3.future>=2.5.900,<3",
     "wassima>=1.0.1,<2",
     "kiss_headers>=2,<4",
 ]
@@ -54,7 +54,7 @@ http3 = [
     "urllib3.future[qh3]"
 ]
 ocsp = [
-    "cryptography<42.0.0,>=41.0.0"
+    "cryptography<43.0.0,>=41.0.0"
 ]
 
 [project.urls]

requirements-dev.txt

@@ -1,5 +1,5 @@
 -e .[socks]
-pytest>=2.8.0,<=7.4.2
+pytest>=2.8.0,<=7.4.4
 pytest-cov
 pytest-httpbin==2.0.0
 pytest-asyncio>=0.21.1,<1.0

src/niquests/__version__.py

@@ -9,9 +9,9 @@
 __url__: str = "https://niquests.readthedocs.io"
 
 __version__: str
-__version__ = "3.4.4"
+__version__ = "3.4.5"
 
-__build__: int = 0x030404
+__build__: int = 0x030405
 __author__: str = "Kenneth Reitz"
 __author_email__: str = "me@kennethreitz.org"
 __license__: str = "Apache-2.0"

src/niquests/adapters.py

@@ -753,6 +753,8 @@ def send(
                 multiplexed=multiplexed,
             )
 
+            if hasattr(self.poolmanager.pools, "memorize"):
+                self.poolmanager.pools.memorize(resp_or_promise, conn)
         except (ProtocolError, OSError) as err:
             if "illegal header" in str(err).lower():
                 raise InvalidHeader(err, request=request)
@@ -935,6 +937,9 @@ def on_post_connection(conn_info: ConnectionInfo) -> None:
 
             origin_response.history.pop()
 
+            origin_response._content = False
+            origin_response._content_consumed = False
+
             origin_response.status_code, leaf_response.status_code = (
                 leaf_response.status_code,
                 origin_response.status_code,
@@ -1053,7 +1058,7 @@ def gather(self, *responses: Response, max_fetch: int | None = None) -> None:
 
                     next_resp = self._future_handler(response, low_resp)
 
-                    if next_resp:
+                    if next_resp is not None:
                         still_redirects.append(next_resp)
 
                 if still_redirects:

src/niquests/sessions.py

@@ -27,7 +27,12 @@
 else:
     from urllib3_future import ConnectionInfo  # type: ignore[assignment]
 
-from ._constant import DEFAULT_RETRIES, READ_DEFAULT_TIMEOUT, WRITE_DEFAULT_TIMEOUT
+from ._constant import (
+    DEFAULT_RETRIES,
+    READ_DEFAULT_TIMEOUT,
+    WRITE_DEFAULT_TIMEOUT,
+    DEFAULT_POOLSIZE,
+)
 from ._typing import (
     BodyType,
     CacheLayerAltSvcType,
@@ -212,6 +217,8 @@ class Session:
         "_disable_ipv6",
         "_disable_http2",
         "_disable_http3",
+        "_pool_connections",
+        "_pool_maxsize",
     ]
 
     def __init__(
@@ -226,7 +233,22 @@ def __init__(
         disable_http3: bool = False,
         disable_ipv6: bool = False,
         disable_ipv4: bool = False,
+        pool_connections: int = DEFAULT_POOLSIZE,
+        pool_maxsize: int = DEFAULT_POOLSIZE,
     ):
+        """
+        :param resolver: Specify a DNS resolver that should be used within this Session.
+        :param source_address: Bind Session to a specific network adapter and/or port so that all outgoing requests.
+        :param quic_cache_layer: Provide an external cache mechanism to store HTTP/3 host capabilities.
+        :param retries: Configure a number of times a request must be automatically retried before giving up.
+        :param multiplexed: Enable or disable concurrent request when the remote host support HTTP/2 onward.
+        :param disable_http2: Toggle to disable negotiating HTTP/2 with remote peers.
+        :param disable_http3: Toggle to disable negotiating HTTP/3 with remote peers.
+        :param disable_ipv6: Toggle to disable using IPv6 even if the remote host supports IPv6.
+        :param disable_ipv4: Toggle to disable using IPv4 even if the remote host supports IPv4.
+        :param pool_connections: Number of concurrent hosts to be handled by this Session at a maximum.
+        :param pool_maxsize: Maximum number of concurrent connections per (single) host at a time.
+        """
         if [disable_ipv4, disable_ipv6].count(True) == 2:
             raise RuntimeError("Cannot disable both IPv4 and IPv6")
 
@@ -283,6 +305,9 @@ def __init__(
         self._disable_ipv4 = disable_ipv4
         self._disable_ipv6 = disable_ipv6
 
+        self._pool_connections = pool_connections
+        self._pool_maxsize = pool_maxsize
+
         #: SSL Verification default.
         #: Defaults to `True`, requiring requests to verify the TLS certificate at the
         #: remote end.
@@ -335,6 +360,8 @@ def __init__(
                 source_address=source_address,
                 disable_ipv4=disable_ipv4,
                 disable_ipv6=disable_ipv6,
+                pool_connections=pool_connections,
+                pool_maxsize=pool_maxsize,
             ),
         )
         self.mount(
@@ -345,6 +372,8 @@ def __init__(
                 source_address=source_address,
                 disable_ipv4=disable_ipv4,
                 disable_ipv6=disable_ipv6,
+                pool_connections=pool_connections,
+                pool_maxsize=pool_maxsize,
             ),
         )
 
@@ -1106,6 +1135,8 @@ def handle_upload_progress(
                     source_address=self.source_address,
                     disable_ipv4=self._disable_ipv4,
                     disable_ipv6=self._disable_ipv6,
+                    pool_connections=self._pool_connections,
+                    pool_maxsize=self._pool_maxsize,
                 ),
             )
             self.mount(
@@ -1116,6 +1147,8 @@ def handle_upload_progress(
                     source_address=self.source_address,
                     disable_ipv4=self._disable_ipv4,
                     disable_ipv6=self._disable_ipv6,
+                    pool_connections=self._pool_connections,
+                    pool_maxsize=self._pool_maxsize,
                 ),
             )
 
@@ -1315,6 +1348,8 @@ def __setstate__(self, state):
                 disable_ipv4=self._disable_ipv4,
                 disable_ipv6=self._disable_ipv6,
                 resolver=self.resolver,
+                pool_connections=self._pool_connections,
+                pool_maxsize=self._pool_maxsize,
             ),
         )
         self.mount(
@@ -1325,6 +1360,8 @@ def __setstate__(self, state):
                 disable_ipv4=self._disable_ipv4,
                 disable_ipv6=self._disable_ipv6,
                 resolver=self.resolver,
+                pool_connections=self._pool_connections,
+                pool_maxsize=self._pool_maxsize,
             ),
         )
 

tests/test_multiplexed.py

@@ -3,7 +3,6 @@
 import pytest
 
 from niquests import Session
-from niquests.exceptions import MultiplexingError
 
 
 @pytest.mark.usefixtures("requires_wan")
@@ -34,6 +33,16 @@ def test_redirect_with_multiplexed(self):
             assert resp.url == "https://pie.dev/get"
             assert len(resp.history) == 3
 
+    def test_redirect_with_multiplexed_direct_access(self):
+        with Session(multiplexed=True) as s:
+            resp = s.get("https://pie.dev/redirect/3")
+            assert resp.lazy
+
+            assert resp.status_code == 200
+            assert resp.url == "https://pie.dev/get"
+            assert len(resp.history) == 3
+            assert resp.json()
+
     def test_lazy_access_sync_mode(self):
         with Session(multiplexed=True) as s:
             resp = s.get("https://pie.dev/headers")
@@ -96,7 +105,7 @@ def test_one_at_a_time(self):
 
             assert len(list(filter(lambda r: r.lazy, responses))) == 0
 
-    def test_early_close_error(self):
+    def test_early_close_no_error(self):
         responses = []
 
         with Session(multiplexed=True) as s:
@@ -105,6 +114,6 @@ def test_early_close_error(self):
 
             assert all(r.lazy for r in responses)
 
-        with pytest.raises(MultiplexingError) as exc:
-            responses[0].json()
-            assert "Did you close the session too early?" in exc.value.args[0]
+        # since urllib3.future 2.5, the scheduler ensure we kept track of ongoing request even if pool is
+        # shutdown.
+        assert all([r.json() for r in responses])