From 70624ffc819c29d842cb68fd86f42eac95421b10 Mon Sep 17 00:00:00 2001 From: royjr Date: Sun, 17 Dec 2023 19:25:25 -0500 Subject: [PATCH] dependency: remove pycurl package (#30771) * Update qcomgpsd.py * Update url_file.py * remove pycurl * requests -> urllib3 * unused * redundant * fix import * Revert "requests -> urllib3" This reverts commit 7ca39e618917986f50356519d2795a432e3a5280. * headless * fix trail * use requests.exceptions.RequestException * use fp.tell * fix indents * reorder imports * change timeout * fix debug timing * remove exception * add timeout * missing headers * move to constructor * move import * unused import * fix debug * try * no retries --- poetry.lock | 23 +----------- pyproject.toml | 2 -- system/qcomgpsd/qcomgpsd.py | 31 ++++++---------- tools/lib/url_file.py | 70 ++++++++++++------------------------- 4 files changed, 34 insertions(+), 92 deletions(-) diff --git a/poetry.lock b/poetry.lock index 3ad023cdd78603..477cbac64c54ef 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3206,16 +3206,6 @@ files = [ {file = "pycryptodome-3.19.0.tar.gz", hash = "sha256:bc35d463222cdb4dbebd35e0784155c81e161b9284e567e7e933d722e533331e"}, ] -[[package]] -name = "pycurl" -version = "7.45.2" -description = "PycURL -- A Python Interface To The cURL library" -optional = false -python-versions = ">=3.5" -files = [ - {file = "pycurl-7.45.2.tar.gz", hash = "sha256:5730590be0271364a5bddd9e245c9cc0fb710c4cbacbdd95264a3122d23224ca"}, -] - [[package]] name = "pyee" version = "11.1.0" @@ -4655,17 +4645,6 @@ notebook = ["ipywidgets (>=6)"] slack = ["slack-sdk"] telegram = ["requests"] -[[package]] -name = "types-pycurl" -version = "7.45.2.5" -description = "Typing stubs for pycurl" -optional = false -python-versions = "*" -files = [ - {file = "types-pycurl-7.45.2.5.tar.gz", hash = "sha256:cc07e4b1e388b98e1a935bc12ef33eec8e7294496d0a81ed50c46a31bf5df5ee"}, - {file = "types_pycurl-7.45.2.5-py3-none-any.whl", hash = "sha256:e433f13a266245be770018eabc6a50139ea26150f75df5b0ee698d291575582b"}, -] - [[package]] name = "types-requests" version = "2.31.0.10" @@ -4902,4 +4881,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "~3.11" -content-hash = "eeb5efb6161eaec95fb863ddc9ec08afb540ed3cd5d521646a6a759ddf33ab2e" +content-hash = "26e963a3e9236b0e29466a0bca243215395a780c88485e9c17f6518969334890" diff --git a/pyproject.toml b/pyproject.toml index 98091e00886220..a519670d489df8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -110,7 +110,6 @@ polyline = "*" # these should be removed markdown-it-py = "*" timezonefinder = "*" -pycurl = "*" setproctitle = "*" @@ -153,7 +152,6 @@ sphinx-rtd-theme = "*" sphinx-sitemap = "*" tabulate = "*" tenacity = "*" -types-pycurl = "*" types-requests = "*" types-tabulate = "*" tqdm = "*" diff --git a/system/qcomgpsd/qcomgpsd.py b/system/qcomgpsd/qcomgpsd.py index 0105d4074f64f7..3f723502340e61 100755 --- a/system/qcomgpsd/qcomgpsd.py +++ b/system/qcomgpsd/qcomgpsd.py @@ -5,7 +5,7 @@ import itertools import math import time -import pycurl +import requests import shutil import subprocess import datetime @@ -102,27 +102,18 @@ def gps_enabled() -> bool: def download_assistance(): try: - c = pycurl.Curl() - c.setopt(pycurl.URL, ASSISTANCE_URL) - c.setopt(pycurl.NOBODY, 1) - c.setopt(pycurl.CONNECTTIMEOUT, 2) - c.perform() - bytes_n = c.getinfo(pycurl.CONTENT_LENGTH_DOWNLOAD) - c.close() - if bytes_n > 1e5: - cloudlog.error("Qcom assistance data larger than expected") - return + response = requests.get(ASSISTANCE_URL, timeout=5, stream=True) with open(ASSIST_DATA_FILE_DOWNLOAD, 'wb') as fp: - c = pycurl.Curl() - c.setopt(pycurl.URL, ASSISTANCE_URL) - c.setopt(pycurl.CONNECTTIMEOUT, 5) - - c.setopt(pycurl.WRITEDATA, fp) - c.perform() - c.close() - os.rename(ASSIST_DATA_FILE_DOWNLOAD, ASSIST_DATA_FILE) - except pycurl.error: + for chunk in response.iter_content(chunk_size=8192): + fp.write(chunk) + if fp.tell() > 1e5: + cloudlog.error("Qcom assistance data larger than expected") + return + + os.rename(ASSIST_DATA_FILE_DOWNLOAD, ASSIST_DATA_FILE) + + except requests.exceptions.RequestException: cloudlog.exception("Failed to download assistance file") return diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py index a54552d2411ee1..3fb2135c088ee6 100644 --- a/tools/lib/url_file.py +++ b/tools/lib/url_file.py @@ -1,10 +1,11 @@ import os import time import threading -import pycurl from hashlib import sha256 -from io import BytesIO +from urllib3 import PoolManager, Retry +from urllib3.util import Timeout from tenacity import retry, wait_random_exponential, stop_after_attempt + from openpilot.common.file_helpers import atomic_write_in_dir from openpilot.system.hardware.hw import Paths # Cache chunk size @@ -35,13 +36,14 @@ def __init__(self, url, debug=False, cache=None): if cache is not None: self._force_download = not cache - try: - self._curl = self._tlocal.curl - except AttributeError: - self._curl = self._tlocal.curl = pycurl.Curl() if not self._force_download: os.makedirs(Paths.download_cache_root(), exist_ok=True) + try: + self._http_client = URLFile._tlocal.http_client + except AttributeError: + self._http_client = URLFile._tlocal.http_client = PoolManager() + def __enter__(self): return self @@ -53,17 +55,10 @@ def __exit__(self, exc_type, exc_value, traceback): @retry(wait=wait_random_exponential(multiplier=1, max=5), stop=stop_after_attempt(3), reraise=True) def get_length_online(self): - c = self._curl - c.reset() - c.setopt(pycurl.NOSIGNAL, 1) - c.setopt(pycurl.TIMEOUT_MS, 500000) - c.setopt(pycurl.FOLLOWLOCATION, True) - c.setopt(pycurl.URL, self._url) - c.setopt(c.NOBODY, 1) - c.perform() - length = int(c.getinfo(c.CONTENT_LENGTH_DOWNLOAD)) - c.reset() - return length + timeout = Timeout(connect=50.0, read=500.0) + response = self._http_client.request('HEAD', self._url, timeout=timeout, preload_content=False) + length = response.headers.get('content-length', 0) + return int(length) def get_length(self): if self._length is not None: @@ -117,7 +112,7 @@ def read(self, ll=None): @retry(wait=wait_random_exponential(multiplier=1, max=5), stop=stop_after_attempt(3), reraise=True) def read_aux(self, ll=None): download_range = False - headers = ["Connection: keep-alive"] + headers = {'Connection': 'keep-alive'} if self._pos != 0 or ll is not None: if ll is None: end = self.get_length() - 1 @@ -125,50 +120,29 @@ def read_aux(self, ll=None): end = min(self._pos + ll, self.get_length()) - 1 if self._pos >= end: return b"" - headers.append(f"Range: bytes={self._pos}-{end}") + headers['Range'] = f"bytes={self._pos}-{end}" download_range = True - dats = BytesIO() - c = self._curl - c.setopt(pycurl.URL, self._url) - c.setopt(pycurl.WRITEDATA, dats) - c.setopt(pycurl.NOSIGNAL, 1) - c.setopt(pycurl.TIMEOUT_MS, 500000) - c.setopt(pycurl.HTTPHEADER, headers) - c.setopt(pycurl.FOLLOWLOCATION, True) - if self._debug: - print("downloading", self._url) - - def header(x): - if b'MISS' in x: - print(x.strip()) - - c.setopt(pycurl.HEADERFUNCTION, header) - - def test(debug_type, debug_msg): - print(" debug(%d): %s" % (debug_type, debug_msg.strip())) - - c.setopt(pycurl.VERBOSE, 1) - c.setopt(pycurl.DEBUGFUNCTION, test) t1 = time.time() - c.perform() + timeout = Timeout(connect=50.0, read=500.0) + response = self._http_client.request('GET', self._url, timeout=timeout, preload_content=False, headers=headers) + ret = response.data if self._debug: t2 = time.time() if t2 - t1 > 0.1: - print(f"get {self._url} {headers!r} {t2 - t1:.f} slow") + print(f"get {self._url} {headers!r} {t2 - t1:.3f} slow") - response_code = c.getinfo(pycurl.RESPONSE_CODE) + response_code = response.status if response_code == 416: # Requested Range Not Satisfiable - raise URLFileException(f"Error, range out of bounds {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}") + raise URLFileException(f"Error, range out of bounds {response_code} {headers} ({self._url}): {repr(ret)[:500]}") if download_range and response_code != 206: # Partial Content - raise URLFileException(f"Error, requested range but got unexpected response {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}") + raise URLFileException(f"Error, requested range but got unexpected response {response_code} {headers} ({self._url}): {repr(ret)[:500]}") if (not download_range) and response_code != 200: # OK - raise URLFileException(f"Error {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}") + raise URLFileException(f"Error {response_code} {headers} ({self._url}): {repr(ret)[:500]}") - ret = dats.getvalue() self._pos += len(ret) return ret