Skip to content

read timeout values from config (or fallback to default) #25

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Dec 23, 2022
Merged

read timeout values from config (or fallback to default) #25

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
208b8b5
read timeout values from config (or fallback to default)
dtrifiro Dec 20, 2022
a55bb3b
fix mypy error
dtrifiro Dec 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 21 additions & 6 deletions dvc_http/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import threading
from getpass import getpass
from typing import BinaryIO, Union
from typing import BinaryIO, Optional, Union

from dvc_objects.fs.base import AnyFSPath, FileSystem
from dvc_objects.fs.callbacks import DEFAULT_CALLBACK, Callback
Expand Down Expand Up @@ -88,6 +88,15 @@ def _prepare_credentials(self, **config):
# The connector should not be owned by aiohttp.ClientSession since
# it is closed by fsspec (HTTPFileSystem.close_session)
client_kwargs["connector_owner"] = False
client_kwargs["connect_timeout"] = config.get(
"connect_timeout", self.REQUEST_TIMEOUT
)
client_kwargs["sock_connect_timeout"] = config.get(
"sock_connect_timeout", self.REQUEST_TIMEOUT
)
client_kwargs["sock_read_timeout"] = config.get(
"sock_read_timeout", self.REQUEST_TIMEOUT
)

# Allow reading proxy configurations from the environment.
client_kwargs["trust_env"] = True
Expand All @@ -96,7 +105,13 @@ def _prepare_credentials(self, **config):
self.upload_method = config.get("method", "POST")
return credentials

async def get_client(self, **kwargs):
async def get_client(
self,
connect_timeout: Optional[float],
sock_connect_timeout: Optional[float],
sock_read_timeout: Optional[float],
**kwargs,
):
import aiohttp
from aiohttp_retry import ExponentialRetry

Expand All @@ -106,7 +121,7 @@ async def get_client(self, **kwargs):
attempts=self.SESSION_RETRIES,
factor=self.SESSION_BACKOFF_FACTOR,
max_timeout=self.REQUEST_TIMEOUT,
exceptions=[aiohttp.ClientError],
exceptions={aiohttp.ClientError},
)

# The default timeout for the aiohttp is 300 seconds
Expand All @@ -116,9 +131,9 @@ async def get_client(self, **kwargs):
# time that is spent when connecting to the remote server.
kwargs["timeout"] = aiohttp.ClientTimeout(
total=None,
connect=self.REQUEST_TIMEOUT,
sock_connect=self.REQUEST_TIMEOUT,
sock_read=self.REQUEST_TIMEOUT,
connect=connect_timeout,
sock_connect=sock_connect_timeout,
sock_read=sock_read_timeout,
)

return ReadOnlyRetryClient(**kwargs)
Expand Down