From 49293d12f40fb8f690cb0c174616980b0816d7c7 Mon Sep 17 00:00:00 2001 From: royjr Date: Sat, 16 Dec 2023 21:07:29 -0500 Subject: [PATCH 01/25] Update qcomgpsd.py --- system/qcomgpsd/qcomgpsd.py | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/system/qcomgpsd/qcomgpsd.py b/system/qcomgpsd/qcomgpsd.py index 0105d4074f64f7..8fea2dfb76cd40 100755 --- a/system/qcomgpsd/qcomgpsd.py +++ b/system/qcomgpsd/qcomgpsd.py @@ -5,7 +5,8 @@ import itertools import math import time -import pycurl +import requests +from requests.exceptions import RequestException import shutil import subprocess import datetime @@ -102,27 +103,21 @@ def gps_enabled() -> bool: def download_assistance(): try: - c = pycurl.Curl() - c.setopt(pycurl.URL, ASSISTANCE_URL) - c.setopt(pycurl.NOBODY, 1) - c.setopt(pycurl.CONNECTTIMEOUT, 2) - c.perform() - bytes_n = c.getinfo(pycurl.CONTENT_LENGTH_DOWNLOAD) - c.close() + response = requests.head(ASSISTANCE_URL, timeout=2) + bytes_n = int(response.headers.get('content-length', 0)) + if bytes_n > 1e5: cloudlog.error("Qcom assistance data larger than expected") return + response = requests.get(ASSISTANCE_URL, timeout=5, stream=True) with open(ASSIST_DATA_FILE_DOWNLOAD, 'wb') as fp: - c = pycurl.Curl() - c.setopt(pycurl.URL, ASSISTANCE_URL) - c.setopt(pycurl.CONNECTTIMEOUT, 5) - - c.setopt(pycurl.WRITEDATA, fp) - c.perform() - c.close() - os.rename(ASSIST_DATA_FILE_DOWNLOAD, ASSIST_DATA_FILE) - except pycurl.error: + for chunk in response.iter_content(chunk_size=8192): + fp.write(chunk) + + os.rename(ASSIST_DATA_FILE_DOWNLOAD, ASSIST_DATA_FILE) + + except RequestException: cloudlog.exception("Failed to download assistance file") return From 3560dc3db6695c4d1cdc731bc697d7fb10681553 Mon Sep 17 00:00:00 2001 From: royjr Date: Sun, 17 Dec 2023 02:31:07 -0500 Subject: [PATCH 02/25] Update url_file.py --- tools/lib/url_file.py | 75 +++++++++++++++++-------------------------- 1 file changed, 30 insertions(+), 45 deletions(-) diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py index a54552d2411ee1..1adbf56d98b05c 100644 --- a/tools/lib/url_file.py +++ b/tools/lib/url_file.py @@ -1,12 +1,15 @@ import os import time import threading -import pycurl from hashlib import sha256 from io import BytesIO from tenacity import retry, wait_random_exponential, stop_after_attempt from openpilot.common.file_helpers import atomic_write_in_dir from openpilot.system.hardware.hw import Paths +from urllib3 import PoolManager, Retry +from urllib3.util import Timeout +from urllib3.util.retry import Retry +import urllib3 # Cache chunk size K = 1000 CHUNK_SIZE = 1000 * K @@ -35,13 +38,14 @@ def __init__(self, url, debug=False, cache=None): if cache is not None: self._force_download = not cache - try: - self._curl = self._tlocal.curl - except AttributeError: - self._curl = self._tlocal.curl = pycurl.Curl() if not self._force_download: os.makedirs(Paths.download_cache_root(), exist_ok=True) + def _get_http_client(self): + if not hasattr(self._tlocal, 'http_client'): + self._tlocal.http_client = PoolManager() + return self._tlocal.http_client + def __enter__(self): return self @@ -53,17 +57,14 @@ def __exit__(self, exc_type, exc_value, traceback): @retry(wait=wait_random_exponential(multiplier=1, max=5), stop=stop_after_attempt(3), reraise=True) def get_length_online(self): - c = self._curl - c.reset() - c.setopt(pycurl.NOSIGNAL, 1) - c.setopt(pycurl.TIMEOUT_MS, 500000) - c.setopt(pycurl.FOLLOWLOCATION, True) - c.setopt(pycurl.URL, self._url) - c.setopt(c.NOBODY, 1) - c.perform() - length = int(c.getinfo(c.CONTENT_LENGTH_DOWNLOAD)) - c.reset() - return length + retries = Retry(connect=3, read=2, redirect=5) + timeout = Timeout(connect=5.0, read=50.0) + try: + response = self._get_http_client().request('HEAD', self._url, retries=retries, timeout=timeout, preload_content=False) + length = response.headers.get('content-length', 0) + return int(length) + except urllib3.exceptions.HTTPError as e: + raise e def get_length(self): if self._length is not None: @@ -117,7 +118,7 @@ def read(self, ll=None): @retry(wait=wait_random_exponential(multiplier=1, max=5), stop=stop_after_attempt(3), reraise=True) def read_aux(self, ll=None): download_range = False - headers = ["Connection: keep-alive"] + headers = {'Connection': 'keep-alive'} if self._pos != 0 or ll is not None: if ll is None: end = self.get_length() - 1 @@ -125,50 +126,34 @@ def read_aux(self, ll=None): end = min(self._pos + ll, self.get_length()) - 1 if self._pos >= end: return b"" - headers.append(f"Range: bytes={self._pos}-{end}") + headers['Range'] = f"bytes={self._pos}-{end}" download_range = True - dats = BytesIO() - c = self._curl - c.setopt(pycurl.URL, self._url) - c.setopt(pycurl.WRITEDATA, dats) - c.setopt(pycurl.NOSIGNAL, 1) - c.setopt(pycurl.TIMEOUT_MS, 500000) - c.setopt(pycurl.HTTPHEADER, headers) - c.setopt(pycurl.FOLLOWLOCATION, True) + response = self._get_http_client().request( + 'GET', + self._url, + headers=headers, + preload_content=False + ) if self._debug: - print("downloading", self._url) - - def header(x): - if b'MISS' in x: - print(x.strip()) - - c.setopt(pycurl.HEADERFUNCTION, header) - - def test(debug_type, debug_msg): - print(" debug(%d): %s" % (debug_type, debug_msg.strip())) - - c.setopt(pycurl.VERBOSE, 1) - c.setopt(pycurl.DEBUGFUNCTION, test) t1 = time.time() - c.perform() + ret = response.data if self._debug: t2 = time.time() if t2 - t1 > 0.1: print(f"get {self._url} {headers!r} {t2 - t1:.f} slow") - response_code = c.getinfo(pycurl.RESPONSE_CODE) + response_code = response.status if response_code == 416: # Requested Range Not Satisfiable - raise URLFileException(f"Error, range out of bounds {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}") + raise URLFileException(f"Error, range out of bounds {response_code} {headers} ({self._url}): {repr(ret)[:500]}") if download_range and response_code != 206: # Partial Content - raise URLFileException(f"Error, requested range but got unexpected response {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}") + raise URLFileException(f"Error, requested range but got unexpected response {response_code} {headers} ({self._url}): {repr(ret)[:500]}") if (not download_range) and response_code != 200: # OK - raise URLFileException(f"Error {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}") + raise URLFileException(f"Error {response_code} {headers} ({self._url}): {repr(ret)[:500]}") - ret = dats.getvalue() self._pos += len(ret) return ret From f50ecf973dadf6700745ebea357f34ac7580f3dd Mon Sep 17 00:00:00 2001 From: royjr Date: Sun, 17 Dec 2023 02:35:02 -0500 Subject: [PATCH 03/25] remove pycurl --- poetry.lock | 23 +---------------------- pyproject.toml | 2 -- 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/poetry.lock b/poetry.lock index 3ad023cdd78603..477cbac64c54ef 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3206,16 +3206,6 @@ files = [ {file = "pycryptodome-3.19.0.tar.gz", hash = "sha256:bc35d463222cdb4dbebd35e0784155c81e161b9284e567e7e933d722e533331e"}, ] -[[package]] -name = "pycurl" -version = "7.45.2" -description = "PycURL -- A Python Interface To The cURL library" -optional = false -python-versions = ">=3.5" -files = [ - {file = "pycurl-7.45.2.tar.gz", hash = "sha256:5730590be0271364a5bddd9e245c9cc0fb710c4cbacbdd95264a3122d23224ca"}, -] - [[package]] name = "pyee" version = "11.1.0" @@ -4655,17 +4645,6 @@ notebook = ["ipywidgets (>=6)"] slack = ["slack-sdk"] telegram = ["requests"] -[[package]] -name = "types-pycurl" -version = "7.45.2.5" -description = "Typing stubs for pycurl" -optional = false -python-versions = "*" -files = [ - {file = "types-pycurl-7.45.2.5.tar.gz", hash = "sha256:cc07e4b1e388b98e1a935bc12ef33eec8e7294496d0a81ed50c46a31bf5df5ee"}, - {file = "types_pycurl-7.45.2.5-py3-none-any.whl", hash = "sha256:e433f13a266245be770018eabc6a50139ea26150f75df5b0ee698d291575582b"}, -] - [[package]] name = "types-requests" version = "2.31.0.10" @@ -4902,4 +4881,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "~3.11" -content-hash = "eeb5efb6161eaec95fb863ddc9ec08afb540ed3cd5d521646a6a759ddf33ab2e" +content-hash = "26e963a3e9236b0e29466a0bca243215395a780c88485e9c17f6518969334890" diff --git a/pyproject.toml b/pyproject.toml index 98091e00886220..a519670d489df8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -110,7 +110,6 @@ polyline = "*" # these should be removed markdown-it-py = "*" timezonefinder = "*" -pycurl = "*" setproctitle = "*" @@ -153,7 +152,6 @@ sphinx-rtd-theme = "*" sphinx-sitemap = "*" tabulate = "*" tenacity = "*" -types-pycurl = "*" types-requests = "*" types-tabulate = "*" tqdm = "*" From 7ca39e618917986f50356519d2795a432e3a5280 Mon Sep 17 00:00:00 2001 From: royjr Date: Sun, 17 Dec 2023 02:42:40 -0500 Subject: [PATCH 04/25] requests -> urllib3 --- system/qcomgpsd/qcomgpsd.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/system/qcomgpsd/qcomgpsd.py b/system/qcomgpsd/qcomgpsd.py index 8fea2dfb76cd40..35b10b6b3f2925 100755 --- a/system/qcomgpsd/qcomgpsd.py +++ b/system/qcomgpsd/qcomgpsd.py @@ -5,8 +5,8 @@ import itertools import math import time -import requests -from requests.exceptions import RequestException +import urllib3 +from urllib3.exceptions import HTTPError import shutil import subprocess import datetime @@ -102,23 +102,23 @@ def gps_enabled() -> bool: raise Exception("failed to execute QGPS mmcli command") from exc def download_assistance(): + http = urllib3.PoolManager() + try: - response = requests.head(ASSISTANCE_URL, timeout=2) - bytes_n = int(response.headers.get('content-length', 0)) + response = http.request('HEAD', ASSISTANCE_URL) + content_length = int(response.headers.get('content-length', 0)) - if bytes_n > 1e5: + if content_length > 1e5: cloudlog.error("Qcom assistance data larger than expected") return - response = requests.get(ASSISTANCE_URL, timeout=5, stream=True) - with open(ASSIST_DATA_FILE_DOWNLOAD, 'wb') as fp: - for chunk in response.iter_content(chunk_size=8192): - fp.write(chunk) + with http.request('GET', ASSISTANCE_URL, preload_content=False) as resp, open(ASSIST_DATA_FILE_DOWNLOAD, 'wb') as out_file: + shutil.copyfileobj(resp, out_file) os.rename(ASSIST_DATA_FILE_DOWNLOAD, ASSIST_DATA_FILE) - except RequestException: - cloudlog.exception("Failed to download assistance file") + except HTTPError as e: + cloudlog.exception("Failed to download assistance file: {}".format(e)) return def downloader_loop(event): From ed660d273d51e137b6ac16592e8dcb346a1a815b Mon Sep 17 00:00:00 2001 From: royjr Date: Sun, 17 Dec 2023 02:44:05 -0500 Subject: [PATCH 05/25] unused --- tools/lib/url_file.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py index 1adbf56d98b05c..bc3e6398ac4af3 100644 --- a/tools/lib/url_file.py +++ b/tools/lib/url_file.py @@ -2,7 +2,6 @@ import time import threading from hashlib import sha256 -from io import BytesIO from tenacity import retry, wait_random_exponential, stop_after_attempt from openpilot.common.file_helpers import atomic_write_in_dir from openpilot.system.hardware.hw import Paths From f24668743c0123a0fdd0c482c8c85259d19627b5 Mon Sep 17 00:00:00 2001 From: royjr Date: Sun, 17 Dec 2023 02:46:44 -0500 Subject: [PATCH 06/25] redundant --- tools/lib/url_file.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py index bc3e6398ac4af3..9027ec926b7d67 100644 --- a/tools/lib/url_file.py +++ b/tools/lib/url_file.py @@ -7,8 +7,6 @@ from openpilot.system.hardware.hw import Paths from urllib3 import PoolManager, Retry from urllib3.util import Timeout -from urllib3.util.retry import Retry -import urllib3 # Cache chunk size K = 1000 CHUNK_SIZE = 1000 * K From 197142e3f542fea1c98385db48dfe344714ecb31 Mon Sep 17 00:00:00 2001 From: royjr Date: Sun, 17 Dec 2023 03:02:45 -0500 Subject: [PATCH 07/25] fix import --- tools/lib/url_file.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py index 9027ec926b7d67..4f8d7e71fe0c1c 100644 --- a/tools/lib/url_file.py +++ b/tools/lib/url_file.py @@ -5,6 +5,7 @@ from tenacity import retry, wait_random_exponential, stop_after_attempt from openpilot.common.file_helpers import atomic_write_in_dir from openpilot.system.hardware.hw import Paths +import urllib3 from urllib3 import PoolManager, Retry from urllib3.util import Timeout # Cache chunk size From 2f112edb6fa7118036a8a3f13201a6e52c385847 Mon Sep 17 00:00:00 2001 From: royjr Date: Sun, 17 Dec 2023 03:10:51 -0500 Subject: [PATCH 08/25] Revert "requests -> urllib3" This reverts commit 7ca39e618917986f50356519d2795a432e3a5280. --- system/qcomgpsd/qcomgpsd.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/system/qcomgpsd/qcomgpsd.py b/system/qcomgpsd/qcomgpsd.py index 35b10b6b3f2925..8fea2dfb76cd40 100755 --- a/system/qcomgpsd/qcomgpsd.py +++ b/system/qcomgpsd/qcomgpsd.py @@ -5,8 +5,8 @@ import itertools import math import time -import urllib3 -from urllib3.exceptions import HTTPError +import requests +from requests.exceptions import RequestException import shutil import subprocess import datetime @@ -102,23 +102,23 @@ def gps_enabled() -> bool: raise Exception("failed to execute QGPS mmcli command") from exc def download_assistance(): - http = urllib3.PoolManager() - try: - response = http.request('HEAD', ASSISTANCE_URL) - content_length = int(response.headers.get('content-length', 0)) + response = requests.head(ASSISTANCE_URL, timeout=2) + bytes_n = int(response.headers.get('content-length', 0)) - if content_length > 1e5: + if bytes_n > 1e5: cloudlog.error("Qcom assistance data larger than expected") return - with http.request('GET', ASSISTANCE_URL, preload_content=False) as resp, open(ASSIST_DATA_FILE_DOWNLOAD, 'wb') as out_file: - shutil.copyfileobj(resp, out_file) + response = requests.get(ASSISTANCE_URL, timeout=5, stream=True) + with open(ASSIST_DATA_FILE_DOWNLOAD, 'wb') as fp: + for chunk in response.iter_content(chunk_size=8192): + fp.write(chunk) os.rename(ASSIST_DATA_FILE_DOWNLOAD, ASSIST_DATA_FILE) - except HTTPError as e: - cloudlog.exception("Failed to download assistance file: {}".format(e)) + except RequestException: + cloudlog.exception("Failed to download assistance file") return def downloader_loop(event): From 308e1a2c1a2c835c5a6caadafe7cdec234c816f3 Mon Sep 17 00:00:00 2001 From: royjr Date: Sun, 17 Dec 2023 03:33:43 -0500 Subject: [PATCH 09/25] headless --- system/qcomgpsd/qcomgpsd.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/system/qcomgpsd/qcomgpsd.py b/system/qcomgpsd/qcomgpsd.py index 8fea2dfb76cd40..b38d8aa4ddace4 100755 --- a/system/qcomgpsd/qcomgpsd.py +++ b/system/qcomgpsd/qcomgpsd.py @@ -103,16 +103,15 @@ def gps_enabled() -> bool: def download_assistance(): try: - response = requests.head(ASSISTANCE_URL, timeout=2) - bytes_n = int(response.headers.get('content-length', 0)) - - if bytes_n > 1e5: - cloudlog.error("Qcom assistance data larger than expected") - return - response = requests.get(ASSISTANCE_URL, timeout=5, stream=True) + downloaded_size = 0 + with open(ASSIST_DATA_FILE_DOWNLOAD, 'wb') as fp: for chunk in response.iter_content(chunk_size=8192): + downloaded_size += len(chunk) + if downloaded_size > 1e5: + cloudlog.error("Qcom assistance data larger than expected") + return fp.write(chunk) os.rename(ASSIST_DATA_FILE_DOWNLOAD, ASSIST_DATA_FILE) From 3d7f75c4aadf1bcbde30bd82aa0b8ec89c67fa86 Mon Sep 17 00:00:00 2001 From: royjr Date: Sun, 17 Dec 2023 03:42:39 -0500 Subject: [PATCH 10/25] fix trail --- system/qcomgpsd/qcomgpsd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/qcomgpsd/qcomgpsd.py b/system/qcomgpsd/qcomgpsd.py index b38d8aa4ddace4..86afca5a8836d7 100755 --- a/system/qcomgpsd/qcomgpsd.py +++ b/system/qcomgpsd/qcomgpsd.py @@ -104,7 +104,7 @@ def gps_enabled() -> bool: def download_assistance(): try: response = requests.get(ASSISTANCE_URL, timeout=5, stream=True) - downloaded_size = 0 + downloaded_size = 0 with open(ASSIST_DATA_FILE_DOWNLOAD, 'wb') as fp: for chunk in response.iter_content(chunk_size=8192): From e919963c5ec17c65f4c1335d058404b8be754ab9 Mon Sep 17 00:00:00 2001 From: royjr Date: Sun, 17 Dec 2023 13:40:29 -0500 Subject: [PATCH 11/25] use requests.exceptions.RequestException --- system/qcomgpsd/qcomgpsd.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/system/qcomgpsd/qcomgpsd.py b/system/qcomgpsd/qcomgpsd.py index 86afca5a8836d7..b1e3df377f87bc 100755 --- a/system/qcomgpsd/qcomgpsd.py +++ b/system/qcomgpsd/qcomgpsd.py @@ -6,7 +6,6 @@ import math import time import requests -from requests.exceptions import RequestException import shutil import subprocess import datetime @@ -116,7 +115,7 @@ def download_assistance(): os.rename(ASSIST_DATA_FILE_DOWNLOAD, ASSIST_DATA_FILE) - except RequestException: + except requests.exceptions.RequestException: cloudlog.exception("Failed to download assistance file") return From b5d88f811f5506cbac3bb6e328c0303ef6cb75ec Mon Sep 17 00:00:00 2001 From: royjr Date: Sun, 17 Dec 2023 13:43:59 -0500 Subject: [PATCH 12/25] use fp.tell --- system/qcomgpsd/qcomgpsd.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/system/qcomgpsd/qcomgpsd.py b/system/qcomgpsd/qcomgpsd.py index b1e3df377f87bc..3f723502340e61 100755 --- a/system/qcomgpsd/qcomgpsd.py +++ b/system/qcomgpsd/qcomgpsd.py @@ -103,15 +103,13 @@ def gps_enabled() -> bool: def download_assistance(): try: response = requests.get(ASSISTANCE_URL, timeout=5, stream=True) - downloaded_size = 0 with open(ASSIST_DATA_FILE_DOWNLOAD, 'wb') as fp: for chunk in response.iter_content(chunk_size=8192): - downloaded_size += len(chunk) - if downloaded_size > 1e5: + fp.write(chunk) + if fp.tell() > 1e5: cloudlog.error("Qcom assistance data larger than expected") return - fp.write(chunk) os.rename(ASSIST_DATA_FILE_DOWNLOAD, ASSIST_DATA_FILE) From dbcb1843c0c8b67d7eb6a2b44d13e30c9c02edf2 Mon Sep 17 00:00:00 2001 From: royjr Date: Sun, 17 Dec 2023 15:35:24 -0500 Subject: [PATCH 13/25] fix indents --- tools/lib/url_file.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py index 4f8d7e71fe0c1c..109732224e9544 100644 --- a/tools/lib/url_file.py +++ b/tools/lib/url_file.py @@ -40,9 +40,9 @@ def __init__(self, url, debug=False, cache=None): os.makedirs(Paths.download_cache_root(), exist_ok=True) def _get_http_client(self): - if not hasattr(self._tlocal, 'http_client'): - self._tlocal.http_client = PoolManager() - return self._tlocal.http_client + if not hasattr(self._tlocal, 'http_client'): + self._tlocal.http_client = PoolManager() + return self._tlocal.http_client def __enter__(self): return self From 3244daccb483bfcbd4243e1559376a515f52b5d7 Mon Sep 17 00:00:00 2001 From: royjr Date: Sun, 17 Dec 2023 16:23:26 -0500 Subject: [PATCH 14/25] reorder imports --- tools/lib/url_file.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py index 109732224e9544..41d01acab86963 100644 --- a/tools/lib/url_file.py +++ b/tools/lib/url_file.py @@ -2,12 +2,13 @@ import time import threading from hashlib import sha256 -from tenacity import retry, wait_random_exponential, stop_after_attempt -from openpilot.common.file_helpers import atomic_write_in_dir -from openpilot.system.hardware.hw import Paths import urllib3 from urllib3 import PoolManager, Retry from urllib3.util import Timeout + +from tenacity import retry, wait_random_exponential, stop_after_attempt +from openpilot.common.file_helpers import atomic_write_in_dir +from openpilot.system.hardware.hw import Paths # Cache chunk size K = 1000 CHUNK_SIZE = 1000 * K From b82798f56391aaac5deb30cf33b7441b44e1cd84 Mon Sep 17 00:00:00 2001 From: royjr Date: Sun, 17 Dec 2023 16:26:06 -0500 Subject: [PATCH 15/25] change timeout --- tools/lib/url_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py index 41d01acab86963..0756d69093b750 100644 --- a/tools/lib/url_file.py +++ b/tools/lib/url_file.py @@ -57,7 +57,7 @@ def __exit__(self, exc_type, exc_value, traceback): @retry(wait=wait_random_exponential(multiplier=1, max=5), stop=stop_after_attempt(3), reraise=True) def get_length_online(self): retries = Retry(connect=3, read=2, redirect=5) - timeout = Timeout(connect=5.0, read=50.0) + timeout = Timeout(connect=50.0, read=500.0) try: response = self._get_http_client().request('HEAD', self._url, retries=retries, timeout=timeout, preload_content=False) length = response.headers.get('content-length', 0) From 9c13b89bfefdff9fac2c033f9a30116d3c23a78f Mon Sep 17 00:00:00 2001 From: royjr Date: Sun, 17 Dec 2023 16:30:43 -0500 Subject: [PATCH 16/25] fix debug timing --- tools/lib/url_file.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py index 0756d69093b750..7dc74c6884998c 100644 --- a/tools/lib/url_file.py +++ b/tools/lib/url_file.py @@ -128,6 +128,9 @@ def read_aux(self, ll=None): headers['Range'] = f"bytes={self._pos}-{end}" download_range = True + if self._debug: + t1 = time.time() + response = self._get_http_client().request( 'GET', self._url, @@ -135,9 +138,6 @@ def read_aux(self, ll=None): preload_content=False ) - if self._debug: - t1 = time.time() - ret = response.data if self._debug: From 4d4e28d564316a0d6cde8e5ee0fef1efe44a0d9c Mon Sep 17 00:00:00 2001 From: royjr Date: Sun, 17 Dec 2023 16:36:18 -0500 Subject: [PATCH 17/25] remove exception --- tools/lib/url_file.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py index 7dc74c6884998c..b90c0a4cc8786e 100644 --- a/tools/lib/url_file.py +++ b/tools/lib/url_file.py @@ -58,12 +58,9 @@ def __exit__(self, exc_type, exc_value, traceback): def get_length_online(self): retries = Retry(connect=3, read=2, redirect=5) timeout = Timeout(connect=50.0, read=500.0) - try: - response = self._get_http_client().request('HEAD', self._url, retries=retries, timeout=timeout, preload_content=False) - length = response.headers.get('content-length', 0) - return int(length) - except urllib3.exceptions.HTTPError as e: - raise e + response = self._get_http_client().request('HEAD', self._url, retries=retries, timeout=timeout, preload_content=False) + length = response.headers.get('content-length', 0) + return int(length) def get_length(self): if self._length is not None: From f790482d46027efce2ecccd14c5e7f61ee2efa38 Mon Sep 17 00:00:00 2001 From: royjr Date: Sun, 17 Dec 2023 16:38:25 -0500 Subject: [PATCH 18/25] add timeout --- tools/lib/url_file.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py index b90c0a4cc8786e..47482294c821ba 100644 --- a/tools/lib/url_file.py +++ b/tools/lib/url_file.py @@ -128,13 +128,9 @@ def read_aux(self, ll=None): if self._debug: t1 = time.time() - response = self._get_http_client().request( - 'GET', - self._url, - headers=headers, - preload_content=False - ) - + retries = Retry(connect=3, read=2, redirect=5) + timeout = Timeout(connect=50.0, read=500.0) + response = self._get_http_client().request('GET', self._url, retries=retries, timeout=timeout, preload_content=False) ret = response.data if self._debug: From 5681f1b379d9c7ecfa50806c20328a79f28146ba Mon Sep 17 00:00:00 2001 From: royjr Date: Sun, 17 Dec 2023 17:09:13 -0500 Subject: [PATCH 19/25] missing headers --- tools/lib/url_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py index 47482294c821ba..3442fd5a5e5ae2 100644 --- a/tools/lib/url_file.py +++ b/tools/lib/url_file.py @@ -130,7 +130,7 @@ def read_aux(self, ll=None): retries = Retry(connect=3, read=2, redirect=5) timeout = Timeout(connect=50.0, read=500.0) - response = self._get_http_client().request('GET', self._url, retries=retries, timeout=timeout, preload_content=False) + response = self._get_http_client().request('GET', self._url, retries=retries, timeout=timeout, preload_content=False, headers=headers) ret = response.data if self._debug: From 27c72313423f3676bfc15021a30b461c0815834f Mon Sep 17 00:00:00 2001 From: royjr Date: Sun, 17 Dec 2023 17:12:25 -0500 Subject: [PATCH 20/25] move to constructor --- tools/lib/url_file.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py index 3442fd5a5e5ae2..da6d260adb48ae 100644 --- a/tools/lib/url_file.py +++ b/tools/lib/url_file.py @@ -40,10 +40,9 @@ def __init__(self, url, debug=False, cache=None): if not self._force_download: os.makedirs(Paths.download_cache_root(), exist_ok=True) - def _get_http_client(self): - if not hasattr(self._tlocal, 'http_client'): - self._tlocal.http_client = PoolManager() - return self._tlocal.http_client + if not hasattr(URLFile._tlocal, 'http_client'): + URLFile._tlocal.http_client = PoolManager() + self._http_client = URLFile._tlocal.http_client def __enter__(self): return self @@ -58,7 +57,7 @@ def __exit__(self, exc_type, exc_value, traceback): def get_length_online(self): retries = Retry(connect=3, read=2, redirect=5) timeout = Timeout(connect=50.0, read=500.0) - response = self._get_http_client().request('HEAD', self._url, retries=retries, timeout=timeout, preload_content=False) + response = self._http_client.request('HEAD', self._url, retries=retries, timeout=timeout, preload_content=False) length = response.headers.get('content-length', 0) return int(length) @@ -130,7 +129,7 @@ def read_aux(self, ll=None): retries = Retry(connect=3, read=2, redirect=5) timeout = Timeout(connect=50.0, read=500.0) - response = self._get_http_client().request('GET', self._url, retries=retries, timeout=timeout, preload_content=False, headers=headers) + response = self._http_client.request('GET', self._url, retries=retries, timeout=timeout, preload_content=False, headers=headers) ret = response.data if self._debug: From 8ff85c95dc3921d3576b25a4cb7e6a5645ae3f3c Mon Sep 17 00:00:00 2001 From: royjr Date: Sun, 17 Dec 2023 17:17:38 -0500 Subject: [PATCH 21/25] move import --- tools/lib/url_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py index da6d260adb48ae..dc39554ab637fa 100644 --- a/tools/lib/url_file.py +++ b/tools/lib/url_file.py @@ -5,8 +5,8 @@ import urllib3 from urllib3 import PoolManager, Retry from urllib3.util import Timeout - from tenacity import retry, wait_random_exponential, stop_after_attempt + from openpilot.common.file_helpers import atomic_write_in_dir from openpilot.system.hardware.hw import Paths # Cache chunk size From d79db3805b5463837adfd3adf05a2ce1ecd003ef Mon Sep 17 00:00:00 2001 From: royjr Date: Sun, 17 Dec 2023 17:37:08 -0500 Subject: [PATCH 22/25] unused import --- tools/lib/url_file.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py index dc39554ab637fa..6d749f98058d27 100644 --- a/tools/lib/url_file.py +++ b/tools/lib/url_file.py @@ -2,7 +2,6 @@ import time import threading from hashlib import sha256 -import urllib3 from urllib3 import PoolManager, Retry from urllib3.util import Timeout from tenacity import retry, wait_random_exponential, stop_after_attempt From eae40fea0c1a3dabc405e07c88074060e6f28743 Mon Sep 17 00:00:00 2001 From: royjr Date: Sun, 17 Dec 2023 17:59:11 -0500 Subject: [PATCH 23/25] fix debug --- tools/lib/url_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py index 6d749f98058d27..d74d2c2b8b66db 100644 --- a/tools/lib/url_file.py +++ b/tools/lib/url_file.py @@ -134,7 +134,7 @@ def read_aux(self, ll=None): if self._debug: t2 = time.time() if t2 - t1 > 0.1: - print(f"get {self._url} {headers!r} {t2 - t1:.f} slow") + print(f"get {self._url} {headers!r} {t2 - t1:.3f} slow") response_code = response.status if response_code == 416: # Requested Range Not Satisfiable From 1ac426eaf2d946da65d4b11551eb62c49fb6ca62 Mon Sep 17 00:00:00 2001 From: royjr Date: Sun, 17 Dec 2023 18:14:34 -0500 Subject: [PATCH 24/25] try --- tools/lib/url_file.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py index d74d2c2b8b66db..cbf56df0c744a5 100644 --- a/tools/lib/url_file.py +++ b/tools/lib/url_file.py @@ -39,9 +39,10 @@ def __init__(self, url, debug=False, cache=None): if not self._force_download: os.makedirs(Paths.download_cache_root(), exist_ok=True) - if not hasattr(URLFile._tlocal, 'http_client'): - URLFile._tlocal.http_client = PoolManager() - self._http_client = URLFile._tlocal.http_client + try: + self._http_client = URLFile._tlocal.http_client + except AttributeError: + self._http_client = URLFile._tlocal.http_client = PoolManager() def __enter__(self): return self From 01e03f03df652a2b53178502885939458b5f81b3 Mon Sep 17 00:00:00 2001 From: royjr Date: Sun, 17 Dec 2023 19:12:01 -0500 Subject: [PATCH 25/25] no retries --- tools/lib/url_file.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py index cbf56df0c744a5..3fb2135c088ee6 100644 --- a/tools/lib/url_file.py +++ b/tools/lib/url_file.py @@ -55,9 +55,8 @@ def __exit__(self, exc_type, exc_value, traceback): @retry(wait=wait_random_exponential(multiplier=1, max=5), stop=stop_after_attempt(3), reraise=True) def get_length_online(self): - retries = Retry(connect=3, read=2, redirect=5) timeout = Timeout(connect=50.0, read=500.0) - response = self._http_client.request('HEAD', self._url, retries=retries, timeout=timeout, preload_content=False) + response = self._http_client.request('HEAD', self._url, timeout=timeout, preload_content=False) length = response.headers.get('content-length', 0) return int(length) @@ -127,9 +126,8 @@ def read_aux(self, ll=None): if self._debug: t1 = time.time() - retries = Retry(connect=3, read=2, redirect=5) timeout = Timeout(connect=50.0, read=500.0) - response = self._http_client.request('GET', self._url, retries=retries, timeout=timeout, preload_content=False, headers=headers) + response = self._http_client.request('GET', self._url, timeout=timeout, preload_content=False, headers=headers) ret = response.data if self._debug: