From 1574d9aaac9cf1e48616183eff255326606438b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniele=20Trifir=C3=B2?= Date: Mon, 14 Mar 2022 09:08:07 +0100 Subject: [PATCH 1/2] fs.http: force cleanup of closed connections --- dvc/fs/http.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/dvc/fs/http.py b/dvc/fs/http.py index d19372810c..f03ea37b39 100644 --- a/dvc/fs/http.py +++ b/dvc/fs/http.py @@ -81,11 +81,20 @@ def _prepare_credentials(self, **config): f"Auth method {auth_method!r} is not supported." ) + # Force cleanup of closed SSL transports. + # https://github.com/iterative/dvc/issues/7414 + connector_kwargs = {"enable_cleanup_closed": True} + if "ssl_verify" in config: - with fsspec_loop(): - client_kwargs["connector"] = aiohttp.TCPConnector( - ssl=make_context(config["ssl_verify"]) - ) + connector_kwargs.update(ssl=make_context(config["ssl_verify"])) + + with fsspec_loop(): + client_kwargs["connector"] = aiohttp.TCPConnector( + **connector_kwargs + ) + # The connector should not be owned by aiohttp.ClientSession since + # it is closed by fsspec (HTTPFileSystem.close_session) + client_kwargs["connector_owner"] = False # Allow reading proxy configurations from the environment. client_kwargs["trust_env"] = True From 12de7e7f367abfca28b2a1a79b6dbe6200e07a3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniele=20Trifir=C3=B2?= Date: Mon, 14 Mar 2022 13:54:07 +0100 Subject: [PATCH 2/2] fs.http: set sock_read timeout --- dvc/fs/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/fs/http.py b/dvc/fs/http.py index f03ea37b39..934cc6ee6c 100644 --- a/dvc/fs/http.py +++ b/dvc/fs/http.py @@ -123,7 +123,7 @@ async def get_client(self, **kwargs): total=None, connect=self.REQUEST_TIMEOUT, sock_connect=self.REQUEST_TIMEOUT, - sock_read=None, + sock_read=self.REQUEST_TIMEOUT, ) return RetryClient(**kwargs)