diff --git a/robobrowser/browser.py b/robobrowser/browser.py index b67ee98..32e9338 100644 --- a/robobrowser/browser.py +++ b/robobrowser/browser.py @@ -6,14 +6,13 @@ import requests from bs4 import BeautifulSoup from werkzeug import cached_property -from requests.exceptions import RequestException +from requests.packages.urllib3.util.retry import Retry from robobrowser import helpers from robobrowser import exceptions from robobrowser.compat import urlparse from robobrowser.forms.form import Form from robobrowser.cache import RoboHTTPAdapter -from robobrowser.helpers import retry _link_ptn = re.compile(r'^(a|button)$', re.I) @@ -68,7 +67,7 @@ class RoboBrowser(object): def __init__(self, session=None, parser=None, user_agent=None, history=True, timeout=None, allow_redirects=True, cache=False, cache_patterns=None, max_age=None, max_count=None, tries=None, - errors=RequestException, delay=None, multiplier=None): + multiplier=None): self.session = session or requests.Session() @@ -107,10 +106,9 @@ def __init__(self, session=None, parser=None, user_agent=None, # Set up retries if tries: - decorator = retry(tries, errors, delay, multiplier) - self._open, self.open = self.open, decorator(self.open) - self._submit_form, self.submit_form = \ - self.submit_form, decorator(self.submit_form) + retry = Retry(tries, backoff_factor=multiplier) + for protocol in ['http://', 'https://']: + self.session.adapters[protocol].max_retries = retry def __repr__(self): try: diff --git a/robobrowser/helpers.py b/robobrowser/helpers.py index 0ac1fcc..95453d5 100644 --- a/robobrowser/helpers.py +++ b/robobrowser/helpers.py @@ -3,9 +3,6 @@ """ import re -import time -import logging -import functools from bs4 import BeautifulSoup from bs4.element import Tag @@ -91,26 +88,3 @@ def lowercase_attr_names(tag): (key.lower(), value) for key, value in iteritems(tag.attrs) ]) - - -def retry(tries, errors=None, delay=3, multiplier=2, logger=None): - - errors = errors or Exception - logger = logger or logging.getLogger(__name__) - - def decorator(func): - - @functools.wraps(func) - def decorated(*args, **kwargs): - mdelay = delay - for _ in range(tries - 1): - try: - return func(*args, **kwargs) - except errors as error: - logger.exception(error) - time.sleep(mdelay) - mdelay *= multiplier - return func(*args, **kwargs) - return decorated - - return decorator