diff --git a/zulip/zulip/__init__.py b/zulip/zulip/__init__.py index 21c862fad..40d7e7197 100644 --- a/zulip/zulip/__init__.py +++ b/zulip/zulip/__init__.py @@ -94,9 +94,8 @@ def fail(self): super(RandomExponentialBackoff, self).fail() # Exponential growth with ratio sqrt(2); compute random delay # between x and 2x where x is growing exponentially - delay_scale = int(2 ** (self.number_of_retries / 2.0 - 1)) + 1 - delay = min(delay_scale + random.randint(1, delay_scale), self.delay_cap) - message = "Sleeping for %ss [max %s] before retrying." % (delay, delay_scale * 2) + delay = random.random() * min(self.delay_cap, 2 ** self.number_of_retries) + message = "Sleeping for %ss before retrying." % (delay,) try: logger.warning(message) except NameError: @@ -510,13 +509,13 @@ def do_api_query(self, orig_request, url, method="POST", query_state = { 'had_error_retry': False, 'request': request, - 'failures': 0, } # type: Dict[str, Any] + # create Random Exponential Backoff object to perform retries using the method + backoff = RandomExponentialBackoff(timeout_success_equivalent=300) + def error_retry(error_string): # type: (str) -> bool - if not self.retry_on_errors or query_state["failures"] >= 10: - return False if self.verbose: if not query_state["had_error_retry"]: sys.stdout.write("zulip API(%s): connection error%s -- retrying." % @@ -526,9 +525,8 @@ def error_retry(error_string): sys.stdout.write(".") sys.stdout.flush() query_state["request"]["dont_block"] = json.dumps(True) - time.sleep(1) - query_state["failures"] += 1 - return True + backoff.fail() + return backoff.keep_going() def end_error_retry(succeeded): # type: (bool) -> None @@ -629,6 +627,8 @@ def call_on_each_event(self, callback, event_types=None, narrow=None): # type: (Callable[[Dict[str, Any]], None], Optional[List[str]], Optional[List[List[str]]]) -> None if narrow is None: narrow = [] + # Use exponential backoff for registering. + register_backoff = RandomExponentialBackoff(timeout_success_equivalent=300) def do_register(): # type: () -> Tuple[str, int] @@ -641,7 +641,7 @@ def do_register(): if 'error' in res['result']: if self.verbose: print("Server returned error:\n%s" % res['msg']) - time.sleep(1) + register_backoff.fail() else: return (res['queue_id'], res['last_event_id']) @@ -649,6 +649,9 @@ def do_register(): # Make long-polling requests with `get_events`. Once a request # has received an answer, pass it to the callback and before # making a new long-polling request. + + # Use exponential backoff for fetching events from server. + fetch_events_backoff = RandomExponentialBackoff(timeout_success_equivalent=300) while True: if queue_id is None: (queue_id, last_event_id) = do_register() @@ -682,7 +685,7 @@ def do_register(): # Add a pause here to cover against potential bugs in this library # causing a DoS attack against a server when getting errors. # TODO: Make this back off exponentially. - time.sleep(1) + fetch_events_backoff.fail() continue for event in res['events']: