From f83a395a4d4edc1a91e06610b98a43579e10db97 Mon Sep 17 00:00:00 2001
From: "Daniel D. Beck" <me@danieldbeck.com>
Date: Sun, 9 Apr 2017 00:02:23 +0100
Subject: [PATCH] try adding a better request retry approach for issue #2

---
 setup.py       |  1 +
 starnearyou.py | 26 +++++++++++++-------------
 2 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/setup.py b/setup.py
index 875c2e8..6574ef9 100644
--- a/setup.py
+++ b/setup.py
@@ -5,6 +5,7 @@
     version='0.1.dev',
     py_modules=['starnearyou'],
     install_requires=[
+        'backoff',
         'Click',
         'lxml',
         'Pillow',
diff --git a/starnearyou.py b/starnearyou.py
index f1d8d59..2f464d3 100644
--- a/starnearyou.py
+++ b/starnearyou.py
@@ -12,6 +12,7 @@
 import time
 import urllib.parse
 
+import backoff
 import click
 import lxml.html
 from PIL import Image
@@ -152,6 +153,7 @@ def cli(work_dir, tweet, auth_info, logfile, loglevel):
             except twython.exceptions.TwythonError as err:
                 logger.exception("Tweeting failed: %r", err)
                 if attempts < limit:
+                    time.sleep(1)
                     continue
                 else:
                     logger.critical("Tweeting failed %s times, aborting.",
@@ -176,6 +178,9 @@ def configure_logging(filename=None, level=logging.INFO):
     console.setFormatter(console_formatter)
     console.setLevel(logging.WARNING)
     logger.addHandler(console)
+    logging.getLogger('backoff').addHandler(console)
+
+    logging.getLogger('backoff').setLevel(logging.INFO)
 
     # log to file
     if filename is not None:
@@ -187,6 +192,7 @@ def configure_logging(filename=None, level=logging.INFO):
         logfile.setFormatter(file_formatter)
         logfile.setLevel(level)
         logger.addHandler(logfile)
+        logging.getLogger('backoff').addHandler(logfile)
 
 
 # =======================
@@ -233,6 +239,9 @@ def make_sun_gif(work_dir):
 # Image fetching
 # ==============
 
+@backoff.on_exception(backoff.expo,
+                     requests.exceptions.RequestException,
+                     max_tries=8)
 def frame_urls(limit=32):
     """Yield the URLs of frames."""
     sdo_url = SDO_URL_TEMPLATE.format(year=start_time.year,
@@ -241,18 +250,7 @@ def frame_urls(limit=32):
                                       hour=start_time.hour)
     logger.info("Fetching frames index: %s", sdo_url)
 
-    max_tries = 3
-    for attempt in range(max_tries):
-        try:
-            response = requests.get(sdo_url, stream=True, timeout=5 * 60)
-        except requests.exceptions.RequestException:
-            logger.debug("Attempt %d of %d failed", attempt + 1, max_tries)
-            if attempt < max_tries - 1:
-                continue
-            else:
-                raise
-        break
-
+    response = requests.get(sdo_url, stream=True, timeout=5 * 60)
     response.raw.decode_content = True
     logger.debug("Frames index reponse: %s", response.status_code)
 
@@ -265,7 +263,9 @@ def frame_urls(limit=32):
     for link in link_tags[-1 * limit:]:
         yield link.get('href')
 
-
+@backoff.on_exception(backoff.expo,
+                     requests.exceptions.RequestException,
+                     max_tries=8)
 def download_frame(url, download_dir):
     """Download the URL to a given directory, if it doesn't already exist."""
     filename = os.path.join(download_dir, split_url(url))