diff --git a/src/hibp_downloader/__init__.py b/src/hibp_downloader/__init__.py index 14abd01..b15c6f9 100644 --- a/src/hibp_downloader/__init__.py +++ b/src/hibp_downloader/__init__.py @@ -13,6 +13,7 @@ LOGGER_NAME = "hibp-downloader" PWNEDPASSWORDS_API_URL = "https://api.pwnedpasswords.com" +HTTPX_TIMEOUT_SECONDS = 30 LOCAL_CACHE_TTL_DEFAULT = 12 * 3600 MULTIPROCESSING_PROCESSES_DEFAULT = int(cpu_count() if cpu_count() else 4) # type: ignore[arg-type] MULTIPROCESSING_PREFIXES_CHUNK_SIZE = 10 diff --git a/src/hibp_downloader/lib/filedata.py b/src/hibp_downloader/lib/filedata.py index f800573..f47c3f8 100644 --- a/src/hibp_downloader/lib/filedata.py +++ b/src/hibp_downloader/lib/filedata.py @@ -24,7 +24,7 @@ def generate_filepath( ) -> Path: return Path( os.path.join( - base_path, + os.path.expanduser(base_path), hash_type.lower(), prefix[0:2].lower(), prefix[2:4].lower(), @@ -42,12 +42,13 @@ async def append_stringfile(filepath: Path, content: str) -> None: async def save_bytesfile(filepath: Path, content: bytes, timestamp: Union[datetime, None] = None) -> None: - await aiofiles.os.makedirs(os.path.dirname(filepath), exist_ok=True) - async with aiofiles.open(filepath, mode="wb") as f: + full_path = os.path.realpath(os.path.expanduser(filepath)) + await aiofiles.os.makedirs(os.path.dirname(full_path), exist_ok=True) + async with aiofiles.open(full_path, mode="wb") as f: await f.write(content) if timestamp: - os.utime(filepath, times=(timestamp.timestamp(), timestamp.timestamp())) + os.utime(full_path, times=(timestamp.timestamp(), timestamp.timestamp())) async def save_datafile( diff --git a/src/hibp_downloader/lib/http.py b/src/hibp_downloader/lib/http.py index bbad015..564fa36 100644 --- a/src/hibp_downloader/lib/http.py +++ b/src/hibp_downloader/lib/http.py @@ -1,11 +1,15 @@ +import random + import httpx -from hibp_downloader import LOGGER_NAME, __title__, __version__ +from hibp_downloader import HTTPX_TIMEOUT_SECONDS, LOGGER_NAME, __title__, __version__ from hibp_downloader.exceptions import HibpDownloaderException from hibp_downloader.lib.logger import logger_get logger = logger_get(LOGGER_NAME) +__TESTING_RANDOM_ERROR_INJECT_RATE = 0 # manual development testing only + async def httpx_debug_request(request): logger.debug(f"request: {request.method} {request.url}") @@ -39,7 +43,7 @@ async def httpx_binary_response(url, etag=None, method="GET", encoding="gzip", d httpx_client = { "headers": headers, "http2": True, - "timeout": 5, + "timeout": HTTPX_TIMEOUT_SECONDS, "follow_redirects": False, "trust_env": False, } @@ -47,11 +51,14 @@ async def httpx_binary_response(url, etag=None, method="GET", encoding="gzip", d if event_hooks: httpx_client["event_hooks"] = event_hooks + if __TESTING_RANDOM_ERROR_INJECT_RATE and __TESTING_RANDOM_ERROR_INJECT_RATE > random.random(): + url = url.replace("http", "broken") + async with httpx.AsyncClient(**httpx_client) as client: request = client.build_request(method=method, url=url) try: response = await client.send(request=request, stream=True) - except (httpx.ConnectError, httpx.RemoteProtocolError): + except (httpx.ConnectError, httpx.RemoteProtocolError, httpx.HTTPError): raise HibpDownloaderException(f"Unable to establish connection {url}") response.binary = b"".join([part async for part in response.aiter_raw()]) diff --git a/src/hibp_downloader/main.py b/src/hibp_downloader/main.py index 2e9f794..765594b 100644 --- a/src/hibp_downloader/main.py +++ b/src/hibp_downloader/main.py @@ -2,7 +2,7 @@ # Copyright [2022-2023] Threat Patrols Pty Ltd (https://www.threatpatrols.com) # -from signal import SIGINT, signal +from signal import SIGINT, SIGTERM, signal from . import LOGGER_NAME from .lib.app import invoke_app @@ -12,12 +12,11 @@ def entrypoint(): + signal(SIGINT, signal_handler) + signal(SIGTERM, signal_handler) invoke_app() -def sigint_handler(__signal_received, __frame): +def signal_handler(__signal_received, __frame): print("SIGINT received, exiting.") exit() - - -signal(SIGINT, sigint_handler)