From f47cf5423bf9d40e57753002bcaac87fc0549de4 Mon Sep 17 00:00:00 2001 From: Kian-Tat Lim Date: Wed, 2 Oct 2024 14:08:04 -0700 Subject: [PATCH 1/2] Add timeouts and retries. --- python/s3daemon/s3daemon.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/python/s3daemon/s3daemon.py b/python/s3daemon/s3daemon.py index fe4669a..496a242 100644 --- a/python/s3daemon/s3daemon.py +++ b/python/s3daemon/s3daemon.py @@ -27,13 +27,23 @@ import aiobotocore.session import botocore +max_connections = int(os.environ.get("S3DAEMON_MAX_CONNECTIONS", 25)) +connect_timeout = float(os.environ.get("S3DAEMON_CONNECT_TIMEOUT", 5.0)) +max_retries = int(os.environ.get("S3DAEMON_MAX_RETRIES", 2)) + config = botocore.config.Config( - max_pool_connections=25, + max_pool_connections=max_connections, tcp_keepalive=True, + connect_timeout=connect_timeout, s3=dict( payload_signing_enabled=False, addressing_style="path", ), + retries=dict( + total_max_attempts=max_retries, + mode="adaptive", + ), + disable_request_compression=True, ) host = os.environ.get("S3DAEMON_HOST", "localhost") From 07b32676667a1ddad6de4d25997dfc76119aa266 Mon Sep 17 00:00:00 2001 From: Kian-Tat Lim Date: Wed, 2 Oct 2024 14:08:16 -0700 Subject: [PATCH 2/2] Add filename to logging. --- python/s3daemon/s3daemon.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/s3daemon/s3daemon.py b/python/s3daemon/s3daemon.py index 496a242..7409110 100644 --- a/python/s3daemon/s3daemon.py +++ b/python/s3daemon/s3daemon.py @@ -81,9 +81,10 @@ async def handle_client(client, reader, writer): try: await client.put_object(Body=f, Bucket=bucket, Key=key) writer.write(b"Success") + log.info("%f %f sec - %s", start, time.time() - start, filename) except Exception as e: writer.write(bytes(repr(e), "UTF-8")) - log.info("%f %f sec", start, time.time() - start) + log.exception("%f %f sec - %s", start, time.time() - start, filename) async def go():