api: Optimize algorithm of Random Exponential Backoff Class.

Fixes #537.

Author: orientor

zulip/zulip/__init__.py

@@ -56,12 +56,13 @@
 API_VERSTRING = "v1/"
 
 class CountingBackoff(object):
-    def __init__(self, maximum_retries=10, timeout_success_equivalent=None):
-        # type: (int, Optional[float]) -> None
+    def __init__(self, maximum_retries=10, timeout_success_equivalent=None, delay_cap=10.0):
+        # type: (int, Optional[float], float) -> None
         self.number_of_retries = 0
         self.maximum_retries = maximum_retries
         self.timeout_success_equivalent = timeout_success_equivalent
         self.last_attempt_time = 0.0
+        self.delay_cap = delay_cap
 
     def keep_going(self):
         # type: () -> bool
@@ -93,9 +94,9 @@ def fail(self):
         super(RandomExponentialBackoff, self).fail()
         # Exponential growth with ratio sqrt(2); compute random delay
         # between x and 2x where x is growing exponentially
-        delay_scale = int(2 ** (self.number_of_retries / 2.0 - 1)) + 1
-        delay = delay_scale + random.randint(1, delay_scale)
-        message = "Sleeping for %ss [max %s] before retrying." % (delay, delay_scale * 2)
+        delay_base = 0.5
+        delay = random.random() * min(self.delay_cap, delay_base * (2 ** self.number_of_retries))
+        message = "Sleeping for %ss before retrying." % (delay,)
         try:
             logger.warning(message)
         except NameError:
@@ -509,13 +510,10 @@ def do_api_query(self, orig_request, url, method="POST",
         query_state = {
             'had_error_retry': False,
             'request': request,
-            'failures': 0,
         }  # type: Dict[str, Any]
 
         def error_retry(error_string):
-            # type: (str) -> bool
-            if not self.retry_on_errors or query_state["failures"] >= 10:
-                return False
+            # type: (str) -> None
             if self.verbose:
                 if not query_state["had_error_retry"]:
                     sys.stdout.write("zulip API(%s): connection error%s -- retrying." %
@@ -525,14 +523,6 @@ def error_retry(error_string):
                     sys.stdout.write(".")
                 sys.stdout.flush()
             query_state["request"]["dont_block"] = json.dumps(True)
-            delay_cap = 10
-            delay_base = 0.5
-            delay_time = random.random() * delay_cap
-            if query_state["failures"] <= 5:
-                delay_time = random.random() * min(delay_cap, delay_base * (2 ** query_state["failures"]))
-            time.sleep(delay_time)
-            query_state["failures"] += 1
-            return True
 
         def end_error_retry(succeeded):
             # type: (bool) -> None
@@ -541,8 +531,8 @@ def end_error_retry(succeeded):
                     print("Success!")
                 else:
                     print("Failed!")
-
-        while True:
+        backoff = RandomExponentialBackoff(timeout_success_equivalent=300)
+        while backoff.keep_going():
             try:
                 if method == "GET":
                     kwarg = "params"
@@ -565,7 +555,9 @@ def end_error_retry(succeeded):
 
                 # On 50x errors, try again after a short sleep
                 if str(res.status_code).startswith('5'):
-                    if error_retry(" (server %s)" % (res.status_code,)):
+                    error_retry(" (server %s)" % (res.status_code,))
+                    backoff.fail()
+                    if backoff.keep_going():
                         continue
                     # Otherwise fall through and process the python-requests error normally
             except (requests.exceptions.Timeout, requests.exceptions.SSLError) as e:
@@ -591,7 +583,9 @@ def end_error_retry(succeeded):
                     # in an invalid site.
                     raise UnrecoverableNetworkError('cannot connect to server ' + self.base_url)
 
-                if error_retry(""):
+                error_retry("")
+                backoff.fail()
+                if backoff.keep_going():
                     continue
                 end_error_retry(False)
                 return {'msg': "Connection error:\n%s" % traceback.format_exc(),
@@ -615,6 +609,7 @@ def end_error_retry(succeeded):
             end_error_retry(False)
             return {'msg': "Unexpected error from the server", "result": "http-error",
                     "status_code": res.status_code}
+        return {'msg': "Unexpected error from the server", "result": "unexpected-error"}
 
     def call_endpoint(self, url=None, method="POST", request=None,
                       longpolling=False, files=None, timeout=None):