Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add timeout environment variable, fix make test examples in CONTRIBUTION.md #110

Closed
wants to merge 11 commits into from
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,5 @@ copernicusmarine/**/__pycache__/
**/.DS_Store
__debug.py
dist/
venv
.tox
4 changes: 2 additions & 2 deletions CONTRIBUTION.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@ If you ping only one person (or pick only one official reviewer), it is possible

Create a test conda environment:
```
make conda-create-test-env
make create-test-environment
```
Then activate this test environment:
```
conda activate {name_of_the_test_environment}
conda activate copernicusmarine_test
```
Export credentials to local variables (if you don't use `moi`, simply put your own credentials):
```
Expand Down
2 changes: 2 additions & 0 deletions copernicusmarine/catalogue_parser/catalogue_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
)
from copernicusmarine.core_functions.environment_variables import (
COPERNICUSMARINE_MAX_CONCURRENT_REQUESTS,
COPERNICUSMARINE_CONNECTION_TIMEOUT,
)
from copernicusmarine.core_functions.sessions import (
get_configured_aiohttp_session,
Expand Down Expand Up @@ -474,6 +475,7 @@ async def get_json_file(self, url: str) -> dict[str, Any]:
url,
params=construct_query_params_for_marine_data_store_monitoring(),
proxy=self.proxy,
timeout=COPERNICUSMARINE_CONNECTION_TIMEOUT,
) as response:
return await response.json()

Expand Down
2 changes: 1 addition & 1 deletion copernicusmarine/core_functions/credentials_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,7 @@ def _check_credentials_with_cas(username: str, password: str) -> bool:
)
conn_session = get_configured_requests_session()
login_session = conn_session.get(
cmems_cas_login_url, proxies=conn_session.proxies
cmems_cas_login_url, proxies=conn_session.proxies,
)
login_from_html = lxml.html.fromstring(login_session.text)
hidden_elements_from_html = login_from_html.xpath(
Expand Down
2 changes: 2 additions & 0 deletions copernicusmarine/core_functions/environment_variables.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,3 +39,5 @@

PROXY_HTTPS = os.getenv("HTTPS_PROXY", "")
PROXY_HTTP = os.getenv("HTTP_PROXY", "")

COPERNICUSMARINE_CONNECTION_TIMEOUT = int(os.getenv("COPERNICUSMARINE_CONNECTION_TIMEOUT", 600))
10 changes: 9 additions & 1 deletion copernicusmarine/core_functions/sessions.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import ssl
from typing import Any, List, Literal, Optional, Tuple
import functools

import aiohttp
import boto3
Expand All @@ -12,6 +13,7 @@

from copernicusmarine.core_functions.environment_variables import (
COPERNICUSMARINE_DISABLE_SSL_CONTEXT,
COPERNICUSMARINE_CONNECTION_TIMEOUT,
COPERNICUSMARINE_TRUST_ENV,
PROXY_HTTP,
PROXY_HTTPS,
Expand All @@ -35,7 +37,10 @@ def _get_ssl_context() -> Optional[ssl.SSLContext]:
def get_configured_aiohttp_session() -> aiohttp.ClientSession:
nest_asyncio.apply()
connector = aiohttp.TCPConnector(ssl=_get_ssl_context())
return aiohttp.ClientSession(connector=connector, trust_env=TRUST_ENV)
client_timeout = aiohttp.ClientTimeout(total=COPERNICUSMARINE_CONNECTION_TIMEOUT)
return aiohttp.ClientSession(
connector=connector, trust_env=TRUST_ENV, timeout=client_timeout
)


def get_https_proxy() -> Optional[str]:
Expand All @@ -52,6 +57,7 @@ def get_configured_boto3_session(
s3={"addressing_style": "virtual"},
signature_version=botocore.UNSIGNED,
retries={"max_attempts": 10, "mode": "standard"},
read_timeout=COPERNICUSMARINE_CONNECTION_TIMEOUT,
)
s3_session = boto3.Session()
s3_client = s3_session.client(
Expand Down Expand Up @@ -91,4 +97,6 @@ def get_configured_requests_session() -> requests.Session:
)
),
)

session.request = functools.partial(session.request, timeout=COPERNICUSMARINE_CONNECTION_TIMEOUT)
return session
Loading