From 89e6745274d8470624c2e3c147060f31b3621e9a Mon Sep 17 00:00:00 2001 From: Vangelis Banos Date: Wed, 8 Jul 2020 16:48:05 +0000 Subject: [PATCH] Handle RuntimeError Some times when warcprox runs for several days under load it freezes and the last error in the log is: ``` WARNING:warcprox.warcproxy.WarcProxy:exception processing request from ('207.241.225.241', 40738) Traceback (most recent call last): File "/usr/lib/python3.7/socketserver.py", line 316, in _handle_request_noblock self.process_request(request, client_address) File "/opt/spn2/lib/python3.7/site-packages/warcprox/mitmproxy.py", line 641, in process_request self.process_request_thread, request, client_address) File "/usr/lib/python3.7/concurrent/futures/thread.py", line 172, in submit self._adjust_thread_count() File "/usr/lib/python3.7/concurrent/futures/thread.py", line 193, in _adjust_thread_count t.start() File "/usr/lib/python3.7/threading.py", line 852, in start _start_new_thread(self._bootstrap, ()) RuntimeError: can't start new thread ``` The process seems to run but it doesn't respond to any connection, not even `status` requests. We handle this exception and allow it to continue operation. --- warcprox/mitmproxy.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/warcprox/mitmproxy.py b/warcprox/mitmproxy.py index 6b32a404..88f6c1e9 100644 --- a/warcprox/mitmproxy.py +++ b/warcprox/mitmproxy.py @@ -637,13 +637,17 @@ def status(self): def process_request(self, request, client_address): self.active_requests[request] = doublethink.utcnow() - future = self.pool.submit( - self.process_request_thread, request, client_address) - future.add_done_callback( - lambda f: self.active_requests.pop(request, None)) - if future.done(): - # avoid theoretical timing issue, in case process_request_thread - # managed to finish before future.add_done_callback() ran + try: + future = self.pool.submit( + self.process_request_thread, request, client_address) + future.add_done_callback( + lambda f: self.active_requests.pop(request, None)) + if future.done(): + # avoid theoretical timing issue, in case process_request_thread + # managed to finish before future.add_done_callback() ran + self.active_requests.pop(request, None) + except RuntimeError as exc: + self.logger.error("Error processing request %s", str(exc)) self.active_requests.pop(request, None) def get_request(self):