From 458bce56db6beeeb6bba6daec382bf18763ea806 Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Tue, 18 Feb 2020 09:17:38 -0500 Subject: [PATCH 01/31] Request limit in paginated message When you invoke _send_paginated_message, it will reach the Coinbase Pro API's rate limit, when the results have too many pages. This patch adds a sleep interval after each page that will work for both public and private endpoints, regardless of the number of pages. This sleep will inevitably slow down paginated calls, but that's required as the number of pages requested grows without bound, per the Coinbase Pro API's spec's rate limits : https://docs.pro.coinbase.com/#rate-limits An additional improvement would be for the private endpoints to send the new sleep_interval parameter at .2 (as opposed to the default .34). --- cbpro/public_client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index e854a163..39fcde8d 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -5,7 +5,7 @@ # For public requests to the Coinbase exchange import requests - +import time class PublicClient(object): """cbpro public client API. @@ -269,7 +269,7 @@ def _send_message(self, method, endpoint, params=None, data=None): auth=self.auth, timeout=30) return r.json() - def _send_paginated_message(self, endpoint, params=None): + def _send_paginated_message(self, endpoint, params=None, sleep_interval = .34): """ Send API message that results in a paginated response. The paginated responses are abstracted away by making API requests on @@ -300,6 +300,7 @@ def _send_paginated_message(self, endpoint, params=None): results = r.json() for result in results: yield result + time.sleep(sleep_interval) # If there are no more pages, we're done. Otherwise update `after` # param to get next page. # If this request included `before` don't get any more pages - the From 11930c488d9466b52e3601199776c0d131a573d4 Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Tue, 18 Feb 2020 09:23:20 -0500 Subject: [PATCH 02/31] sleep_interval added to paginated calls Added optimization for authenticated calls to reduce the sleep_interval from .34 to .2 for authenticated requests, per the Coinbase API request limit spec at: https://docs.pro.coinbase.com/#rate-limits --- cbpro/authenticated_client.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cbpro/authenticated_client.py b/cbpro/authenticated_client.py index 0c2dc329..804ab6f8 100644 --- a/cbpro/authenticated_client.py +++ b/cbpro/authenticated_client.py @@ -126,7 +126,7 @@ def get_account_history(self, account_id, **kwargs): ] """ endpoint = '/accounts/{}/ledger'.format(account_id) - return self._send_paginated_message(endpoint, params=kwargs) + return self._send_paginated_message(endpoint, params=kwargs, sleep_interval=.2) def get_account_holds(self, account_id, **kwargs): """ Get holds on an account. @@ -171,7 +171,7 @@ def get_account_holds(self, account_id, **kwargs): """ endpoint = '/accounts/{}/holds'.format(account_id) - return self._send_paginated_message(endpoint, params=kwargs) + return self._send_paginated_message(endpoint, params=kwargs, sleep_interval=.2) def place_order(self, product_id, side, order_type, **kwargs): """ Place an order. @@ -579,7 +579,7 @@ def get_orders(self, product_id=None, status=None, **kwargs): params['product_id'] = product_id if status is not None: params['status'] = status - return self._send_paginated_message('/orders', params=params) + return self._send_paginated_message('/orders', params=params, sleep_interval=.2) def get_fills(self, product_id=None, order_id=None, **kwargs): """ Get a list of recent fills. @@ -637,7 +637,7 @@ def get_fills(self, product_id=None, order_id=None, **kwargs): params['order_id'] = order_id params.update(kwargs) - return self._send_paginated_message('/fills', params=params) + return self._send_paginated_message('/fills', params=params, sleep_interval=.2) def get_fundings(self, status=None, **kwargs): """ Every order placed with a margin profile that draws funding @@ -676,7 +676,7 @@ def get_fundings(self, status=None, **kwargs): if status is not None: params['status'] = status params.update(kwargs) - return self._send_paginated_message('/funding', params=params) + return self._send_paginated_message('/funding', params=params, sleep_interval=.2) def repay_funding(self, amount, currency): """ Repay funding. Repays the older funding records first. From 1e463a5062c41e75ec592e09e653ef4efa5700d2 Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Tue, 18 Feb 2020 10:17:09 -0500 Subject: [PATCH 03/31] Decreased performance penalty The location of the time.sleep call in the previous commit caused a huge performance hit. I moved the sleep call to a place where it will only be invoked immediately before the next page is requested. --- cbpro/public_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index 39fcde8d..373a7533 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -300,13 +300,14 @@ def _send_paginated_message(self, endpoint, params=None, sleep_interval = .34): results = r.json() for result in results: yield result - time.sleep(sleep_interval) # If there are no more pages, we're done. Otherwise update `after` # param to get next page. # If this request included `before` don't get any more pages - the # cbpro API doesn't support multiple pages in that case. if not r.headers.get('cb-after') or \ params.get('before') is not None: + time.sleep(sleep_interval) break else: params['after'] = r.headers['cb-after'] + time.sleep(sleep_interval) From 9e6c291ef77d1e091b3bd27842e5d29d6ceffa10 Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Sun, 19 Apr 2020 11:28:50 -0400 Subject: [PATCH 04/31] Added get_profile and get_profiles calls Coinbase has added separate portfolio functionality within a single account. These two new functions add that functionality to CBPro. --- cbpro/authenticated_client.py | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/cbpro/authenticated_client.py b/cbpro/authenticated_client.py index 804ab6f8..4413fc91 100644 --- a/cbpro/authenticated_client.py +++ b/cbpro/authenticated_client.py @@ -173,6 +173,45 @@ def get_account_holds(self, account_id, **kwargs): endpoint = '/accounts/{}/holds'.format(account_id) return self._send_paginated_message(endpoint, params=kwargs, sleep_interval=.2) + def get_profiles(self, **kwargs): + """List Profiles + + List all account profiles (also known as portfolios). + + Returns: + [ + { + "id": "86602c68-306a-4500-ac73-4ce56a91d83c", + "user_id": "5844eceecf7e803e259d0365", + "name": "default", + "active": true, + "is_default": true, + "created_at": "2019-11-18T15:08:40.236309Z" + } + ] + """ + return self._send_message('get', '/profiles') + + def get_profile(self, profile_id, **kwargs): + """Get a Profile + + Get a single profile by profile id. + + Args: + profile_id(str): The profile id to get + + Returns: + { + "id": "86602c68-306a-4500-ac73-4ce56a91d83c", + "user_id": "5844eceecf7e803e259d0365", + "name": "default", + "active": true, + "is_default": true, + "created_at": "2019-11-18T15:08:40.236309Z" + } + """ + return self._send_message('get', '/profile/' + profile_id) + def place_order(self, product_id, side, order_type, **kwargs): """ Place an order. From 069cc43674e4ab683e591bf3f070a94b1804a67a Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Sat, 15 Aug 2020 18:41:06 -0400 Subject: [PATCH 05/31] Made sleep_interval a keyword argument The sleep_interval parameter is now a keyword argument --- cbpro/public_client.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index 373a7533..73eaf91a 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -269,7 +269,7 @@ def _send_message(self, method, endpoint, params=None, data=None): auth=self.auth, timeout=30) return r.json() - def _send_paginated_message(self, endpoint, params=None, sleep_interval = .34): + def _send_paginated_message(self, endpoint, params=None, **kwargs): """ Send API message that results in a paginated response. The paginated responses are abstracted away by making API requests on @@ -287,6 +287,7 @@ def _send_paginated_message(self, endpoint, params=None, sleep_interval = .34): Args: endpoint (str): Endpoint (to be added to base URL) params (Optional[dict]): HTTP request parameters + sleep_interval (Optional[float]): Number of seconds to sleep between paginated calls Yields: dict: API response objects @@ -306,8 +307,11 @@ def _send_paginated_message(self, endpoint, params=None, sleep_interval = .34): # cbpro API doesn't support multiple pages in that case. if not r.headers.get('cb-after') or \ params.get('before') is not None: - time.sleep(sleep_interval) + #If a sleep_interval was sent, use it + if "sleep_interval" in kwargs.keys(): + time.sleep(sleep_interval) break else: params['after'] = r.headers['cb-after'] - time.sleep(sleep_interval) + if "sleep_interval" in kwargs.keys(): + time.sleep(sleep_interval) From 55b48b2757251b60c8e88f544153f115a91c6fb8 Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Sat, 15 Aug 2020 19:02:35 -0400 Subject: [PATCH 06/31] Removed sleep_interval keyword parameters Now uses **kwargs to pass the sleep_interval parameter --- cbpro/authenticated_client.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cbpro/authenticated_client.py b/cbpro/authenticated_client.py index 4413fc91..090135c4 100644 --- a/cbpro/authenticated_client.py +++ b/cbpro/authenticated_client.py @@ -126,7 +126,7 @@ def get_account_history(self, account_id, **kwargs): ] """ endpoint = '/accounts/{}/ledger'.format(account_id) - return self._send_paginated_message(endpoint, params=kwargs, sleep_interval=.2) + return self._send_paginated_message(endpoint, params=kwargs) def get_account_holds(self, account_id, **kwargs): """ Get holds on an account. @@ -171,7 +171,7 @@ def get_account_holds(self, account_id, **kwargs): """ endpoint = '/accounts/{}/holds'.format(account_id) - return self._send_paginated_message(endpoint, params=kwargs, sleep_interval=.2) + return self._send_paginated_message(endpoint, params=kwargs) def get_profiles(self, **kwargs): """List Profiles @@ -618,7 +618,7 @@ def get_orders(self, product_id=None, status=None, **kwargs): params['product_id'] = product_id if status is not None: params['status'] = status - return self._send_paginated_message('/orders', params=params, sleep_interval=.2) + return self._send_paginated_message('/orders', params=params) def get_fills(self, product_id=None, order_id=None, **kwargs): """ Get a list of recent fills. @@ -676,7 +676,7 @@ def get_fills(self, product_id=None, order_id=None, **kwargs): params['order_id'] = order_id params.update(kwargs) - return self._send_paginated_message('/fills', params=params, sleep_interval=.2) + return self._send_paginated_message('/fills', params=params) def get_fundings(self, status=None, **kwargs): """ Every order placed with a margin profile that draws funding @@ -715,7 +715,7 @@ def get_fundings(self, status=None, **kwargs): if status is not None: params['status'] = status params.update(kwargs) - return self._send_paginated_message('/funding', params=params, sleep_interval=.2) + return self._send_paginated_message('/funding', params=params) def repay_funding(self, amount, currency): """ Repay funding. Repays the older funding records first. From e697d11bfab8fc32c0d65ee6475a1199063c7a0d Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Tue, 12 Jan 2021 20:13:32 -0500 Subject: [PATCH 07/31] Trying to handle exception in paginated generator For some reason, sometimes the response is empty. --- cbpro/public_client.py | 44 ++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index 73eaf91a..a4ed7876 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -6,6 +6,9 @@ import requests import time +import sys +import traceback +import ast class PublicClient(object): """cbpro public client API. @@ -296,22 +299,29 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): if params is None: params = dict() url = self.url + endpoint + while True: - r = self.session.get(url, params=params, auth=self.auth, timeout=30) - results = r.json() - for result in results: - yield result - # If there are no more pages, we're done. Otherwise update `after` - # param to get next page. - # If this request included `before` don't get any more pages - the - # cbpro API doesn't support multiple pages in that case. - if not r.headers.get('cb-after') or \ - params.get('before') is not None: - #If a sleep_interval was sent, use it + try: + r = self.session.get(url, params=params, auth=self.auth, timeout=30) + results = r.json() + for result in results: + yield result + # If there are no more pages, we're done. Otherwise update `after` + # param to get next page. + # If this request included `before` don't get any more pages - the + # cbpro API doesn't support multiple pages in that case. + if not r.headers.get('cb-after') or \ + params.get('before') is not None: + #If a sleep_interval was sent, use it + if "sleep_interval" in kwargs.keys(): + time.sleep(sleep_interval) + break + else: + params['after'] = r.headers['cb-after'] + if "sleep_interval" in kwargs.keys(): + time.sleep(sleep_interval) + except ValueError: if "sleep_interval" in kwargs.keys(): - time.sleep(sleep_interval) - break - else: - params['after'] = r.headers['cb-after'] - if "sleep_interval" in kwargs.keys(): - time.sleep(sleep_interval) + time.sleep(sleep_interval) + print(sys.exc_info(), flush=True) + continue From 1b40752151e3264091cb91cacd0a71bc464eb3bd Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Tue, 12 Jan 2021 20:17:03 -0500 Subject: [PATCH 08/31] More pagination exceptions in generator I'm not sure that the internal exceptions are correct --- cbpro/public_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index a4ed7876..1a9b0531 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -320,7 +320,7 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): params['after'] = r.headers['cb-after'] if "sleep_interval" in kwargs.keys(): time.sleep(sleep_interval) - except ValueError: + except (ValueError, AttributeError): if "sleep_interval" in kwargs.keys(): time.sleep(sleep_interval) print(sys.exc_info(), flush=True) From 6256edf087fcc1344d63cdba83862bb2e610f681 Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Tue, 12 Jan 2021 20:23:55 -0500 Subject: [PATCH 09/31] Sending troubleshooting output I want to see the result. --- cbpro/public_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index 1a9b0531..3eb5d8de 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -305,6 +305,7 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): r = self.session.get(url, params=params, auth=self.auth, timeout=30) results = r.json() for result in results: + print(result) yield result # If there are no more pages, we're done. Otherwise update `after` # param to get next page. From b07dfec48f58fb25f23d1de6cad3a0beb0dabcf1 Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Tue, 12 Jan 2021 21:48:23 -0500 Subject: [PATCH 10/31] Fixed indentation issue in _send_paginated_message Except needed its indentation fixed. --- cbpro/public_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index 3eb5d8de..df6a4e1f 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -321,7 +321,7 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): params['after'] = r.headers['cb-after'] if "sleep_interval" in kwargs.keys(): time.sleep(sleep_interval) - except (ValueError, AttributeError): + except (ValueError, AttributeError): if "sleep_interval" in kwargs.keys(): time.sleep(sleep_interval) print(sys.exc_info(), flush=True) From 8f9e43a4e6fd52dcda57919b95d5606367e6f730 Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Tue, 12 Jan 2021 21:54:01 -0500 Subject: [PATCH 11/31] Added more output to the exception --- cbpro/public_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index df6a4e1f..b574fecc 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -323,6 +323,7 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): time.sleep(sleep_interval) except (ValueError, AttributeError): if "sleep_interval" in kwargs.keys(): + print(result) time.sleep(sleep_interval) print(sys.exc_info(), flush=True) continue From a9827a286b8d73366fcc5c12b261a2b214f13e57 Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Tue, 12 Jan 2021 22:23:09 -0500 Subject: [PATCH 12/31] Added check for empty result Is the Coinbase API returning empty results? --- cbpro/public_client.py | 48 +++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index b574fecc..8313fc9d 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -301,29 +301,29 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): url = self.url + endpoint while True: - try: - r = self.session.get(url, params=params, auth=self.auth, timeout=30) - results = r.json() - for result in results: - print(result) + #try: + r = self.session.get(url, params=params, auth=self.auth, timeout=30) + results = r.json() + for result in results: + if result != []: yield result - # If there are no more pages, we're done. Otherwise update `after` - # param to get next page. - # If this request included `before` don't get any more pages - the - # cbpro API doesn't support multiple pages in that case. - if not r.headers.get('cb-after') or \ - params.get('before') is not None: - #If a sleep_interval was sent, use it - if "sleep_interval" in kwargs.keys(): - time.sleep(sleep_interval) - break - else: - params['after'] = r.headers['cb-after'] - if "sleep_interval" in kwargs.keys(): - time.sleep(sleep_interval) - except (ValueError, AttributeError): + # If there are no more pages, we're done. Otherwise update `after` + # param to get next page. + # If this request included `before` don't get any more pages - the + # cbpro API doesn't support multiple pages in that case. + if not r.headers.get('cb-after') or \ + params.get('before') is not None: + #If a sleep_interval was sent, use it if "sleep_interval" in kwargs.keys(): - print(result) - time.sleep(sleep_interval) - print(sys.exc_info(), flush=True) - continue + time.sleep(sleep_interval) + break + else: + params['after'] = r.headers['cb-after'] + if "sleep_interval" in kwargs.keys(): + time.sleep(sleep_interval) + #except (ValueError, AttributeError): + # if "sleep_interval" in kwargs.keys(): + # print(result) + # time.sleep(sleep_interval) + # print(sys.exc_info(), flush=True) + # continue From d270a983a340b0147e03fe07e95b9cbacae2445a Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Tue, 12 Jan 2021 22:47:45 -0500 Subject: [PATCH 13/31] Trying to remove empty results This time, I'm using an empty string. --- cbpro/public_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index 8313fc9d..f1d592b1 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -305,7 +305,7 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): r = self.session.get(url, params=params, auth=self.auth, timeout=30) results = r.json() for result in results: - if result != []: + if result != "": yield result # If there are no more pages, we're done. Otherwise update `after` # param to get next page. From d8f98333eb4f38c4a09107aed2acc294dacf1a79 Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Wed, 13 Jan 2021 13:38:47 -0500 Subject: [PATCH 14/31] Added output every time I sleep Still troubleshooting --- cbpro/public_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index f1d592b1..2627159f 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -315,11 +315,13 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): params.get('before') is not None: #If a sleep_interval was sent, use it if "sleep_interval" in kwargs.keys(): + print("Sleeping") time.sleep(sleep_interval) break else: params['after'] = r.headers['cb-after'] if "sleep_interval" in kwargs.keys(): + print("Sleeping") time.sleep(sleep_interval) #except (ValueError, AttributeError): # if "sleep_interval" in kwargs.keys(): From b51ba2a895d334e6609d247d8d7c9506c50d4ad4 Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Wed, 13 Jan 2021 13:41:51 -0500 Subject: [PATCH 15/31] Flushed the print statements I need output --- cbpro/public_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index 2627159f..ca3d4557 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -315,13 +315,13 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): params.get('before') is not None: #If a sleep_interval was sent, use it if "sleep_interval" in kwargs.keys(): - print("Sleeping") + print("Sleeping", flush=True) time.sleep(sleep_interval) break else: params['after'] = r.headers['cb-after'] if "sleep_interval" in kwargs.keys(): - print("Sleeping") + print("Sleeping", flush=True) time.sleep(sleep_interval) #except (ValueError, AttributeError): # if "sleep_interval" in kwargs.keys(): From cb5fd1e838e0c92b42304d580b7b1ce7247b470b Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Wed, 13 Jan 2021 13:43:58 -0500 Subject: [PATCH 16/31] Added sleep higher up Let's see if this takes --- cbpro/public_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index ca3d4557..dd32ed2e 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -305,6 +305,8 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): r = self.session.get(url, params=params, auth=self.auth, timeout=30) results = r.json() for result in results: + print("Sleeping", flush=True) + time.sleep(sleep_interval) if result != "": yield result # If there are no more pages, we're done. Otherwise update `after` From af70242a079ded972aa02692d821683de6776c10 Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Wed, 13 Jan 2021 13:48:09 -0500 Subject: [PATCH 17/31] Is sleep_interval being passed? Looks like the kwarg isn't coming in . . . --- cbpro/public_client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index dd32ed2e..a5b04c38 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -305,8 +305,9 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): r = self.session.get(url, params=params, auth=self.auth, timeout=30) results = r.json() for result in results: - print("Sleeping", flush=True) - time.sleep(sleep_interval) + if "sleep_interval" in kwargs.keys(): + print("Sleeping", flush=True) + time.sleep(sleep_interval) if result != "": yield result # If there are no more pages, we're done. Otherwise update `after` From e7c0ace9b089b020eeb08170701b452126bed675 Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Wed, 13 Jan 2021 13:52:59 -0500 Subject: [PATCH 18/31] Fixed the sleep code . . . Let's see what else crops up . . . --- cbpro/public_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index a5b04c38..1b7a203e 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -307,7 +307,7 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): for result in results: if "sleep_interval" in kwargs.keys(): print("Sleeping", flush=True) - time.sleep(sleep_interval) + time.sleep(kwargs["sleep_interval"]) if result != "": yield result # If there are no more pages, we're done. Otherwise update `after` @@ -319,13 +319,13 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): #If a sleep_interval was sent, use it if "sleep_interval" in kwargs.keys(): print("Sleeping", flush=True) - time.sleep(sleep_interval) + time.sleep(kwargs["sleep_interval"]) break else: params['after'] = r.headers['cb-after'] if "sleep_interval" in kwargs.keys(): print("Sleeping", flush=True) - time.sleep(sleep_interval) + time.sleep(kwargs["sleep_interval"]) #except (ValueError, AttributeError): # if "sleep_interval" in kwargs.keys(): # print(result) From a36c72a01010350046c9bea8420d0c29c2b1de97 Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Wed, 13 Jan 2021 14:50:03 -0500 Subject: [PATCH 19/31] Reinserted exception code Am I getting warmer? --- cbpro/public_client.py | 60 +++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index 1b7a203e..ebb2c87f 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -301,34 +301,34 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): url = self.url + endpoint while True: - #try: - r = self.session.get(url, params=params, auth=self.auth, timeout=30) - results = r.json() - for result in results: + try: + r = self.session.get(url, params=params, auth=self.auth, timeout=30) + results = r.json() + for result in results: + if "sleep_interval" in kwargs.keys(): + print("Sleeping", flush=True) + time.sleep(kwargs["sleep_interval"]) + if result != "": + yield result + # If there are no more pages, we're done. Otherwise update `after` + # param to get next page. + # If this request included `before` don't get any more pages - the + # cbpro API doesn't support multiple pages in that case. + if not r.headers.get('cb-after') or \ + params.get('before') is not None: + #If a sleep_interval was sent, use it + if "sleep_interval" in kwargs.keys(): + print("Sleeping", flush=True) + time.sleep(kwargs["sleep_interval"]) + break + else: + params['after'] = r.headers['cb-after'] + if "sleep_interval" in kwargs.keys(): + print("Sleeping", flush=True) + time.sleep(kwargs["sleep_interval"]) + except (ValueError, AttributeError): if "sleep_interval" in kwargs.keys(): - print("Sleeping", flush=True) - time.sleep(kwargs["sleep_interval"]) - if result != "": - yield result - # If there are no more pages, we're done. Otherwise update `after` - # param to get next page. - # If this request included `before` don't get any more pages - the - # cbpro API doesn't support multiple pages in that case. - if not r.headers.get('cb-after') or \ - params.get('before') is not None: - #If a sleep_interval was sent, use it - if "sleep_interval" in kwargs.keys(): - print("Sleeping", flush=True) - time.sleep(kwargs["sleep_interval"]) - break - else: - params['after'] = r.headers['cb-after'] - if "sleep_interval" in kwargs.keys(): - print("Sleeping", flush=True) - time.sleep(kwargs["sleep_interval"]) - #except (ValueError, AttributeError): - # if "sleep_interval" in kwargs.keys(): - # print(result) - # time.sleep(sleep_interval) - # print(sys.exc_info(), flush=True) - # continue + print(result) + time.sleep(kwargs["sleep_interval"]) + print(sys.exc_info(), flush=True) + continue From f6a2162cc8028d22fb3747339e23ae9b673ad61d Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Wed, 13 Jan 2021 15:10:09 -0500 Subject: [PATCH 20/31] Converted the generated result to a string Maybe that will resolve the issue? --- cbpro/public_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index ebb2c87f..63d77490 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -309,7 +309,7 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): print("Sleeping", flush=True) time.sleep(kwargs["sleep_interval"]) if result != "": - yield result + yield str(result) # If there are no more pages, we're done. Otherwise update `after` # param to get next page. # If this request included `before` don't get any more pages - the From ac0944396095db6d30cfca58188d2048a513c2cf Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Wed, 13 Jan 2021 15:13:34 -0500 Subject: [PATCH 21/31] Removed exception and extra sleep calls Trying to tighten up the code --- cbpro/public_client.py | 52 ++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 30 deletions(-) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index 63d77490..cf215eff 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -301,34 +301,26 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): url = self.url + endpoint while True: - try: - r = self.session.get(url, params=params, auth=self.auth, timeout=30) - results = r.json() - for result in results: - if "sleep_interval" in kwargs.keys(): - print("Sleeping", flush=True) - time.sleep(kwargs["sleep_interval"]) - if result != "": - yield str(result) - # If there are no more pages, we're done. Otherwise update `after` - # param to get next page. - # If this request included `before` don't get any more pages - the - # cbpro API doesn't support multiple pages in that case. - if not r.headers.get('cb-after') or \ - params.get('before') is not None: - #If a sleep_interval was sent, use it - if "sleep_interval" in kwargs.keys(): - print("Sleeping", flush=True) - time.sleep(kwargs["sleep_interval"]) - break - else: - params['after'] = r.headers['cb-after'] - if "sleep_interval" in kwargs.keys(): - print("Sleeping", flush=True) - time.sleep(kwargs["sleep_interval"]) - except (ValueError, AttributeError): + r = self.session.get(url, params=params, auth=self.auth, timeout=30) + results = r.json() + for result in results: if "sleep_interval" in kwargs.keys(): - print(result) - time.sleep(kwargs["sleep_interval"]) - print(sys.exc_info(), flush=True) - continue + time.sleep(kwargs["sleep_interval"]) + if result != "": + yield str(result) + # If there are no more pages, we're done. Otherwise update `after` + # param to get next page. + # If this request included `before` don't get any more pages - the + # cbpro API doesn't support multiple pages in that case. + if not r.headers.get('cb-after') or \ + params.get('before') is not None: + #If a sleep_interval was sent, use it +# if "sleep_interval" in kwargs.keys(): +# print("Sleeping", flush=True) +# time.sleep(kwargs["sleep_interval"]) + break + else: + params['after'] = r.headers['cb-after'] +# if "sleep_interval" in kwargs.keys(): +# print("Sleeping", flush=True) +# time.sleep(kwargs["sleep_interval"]) From 97f53e4eb393981f677ddb3853d2df4ea2c12f8c Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Wed, 13 Jan 2021 15:20:07 -0500 Subject: [PATCH 22/31] How long do I sleep? It doesn't seem to be sleeping long enough . . . --- cbpro/public_client.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index cf215eff..a670b870 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -304,8 +304,8 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): r = self.session.get(url, params=params, auth=self.auth, timeout=30) results = r.json() for result in results: - if "sleep_interval" in kwargs.keys(): - time.sleep(kwargs["sleep_interval"]) +# if "sleep_interval" in kwargs.keys(): +# time.sleep(kwargs["sleep_interval"]) if result != "": yield str(result) # If there are no more pages, we're done. Otherwise update `after` @@ -315,12 +315,12 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): if not r.headers.get('cb-after') or \ params.get('before') is not None: #If a sleep_interval was sent, use it -# if "sleep_interval" in kwargs.keys(): -# print("Sleeping", flush=True) -# time.sleep(kwargs["sleep_interval"]) + if "sleep_interval" in kwargs.keys(): + print("Sleeping", flush=True) + time.sleep(kwargs["sleep_interval"]) break else: params['after'] = r.headers['cb-after'] -# if "sleep_interval" in kwargs.keys(): -# print("Sleeping", flush=True) -# time.sleep(kwargs["sleep_interval"]) + if "sleep_interval" in kwargs.keys(): + print("Sleeping", flush=True) + time.sleep(kwargs["sleep_interval"]) From 5635e4caae251f8ba343916e93b6c056ed47bc8d Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Wed, 13 Jan 2021 15:31:10 -0500 Subject: [PATCH 23/31] Code cleanup for many pages of results The bug should be gone now. --- cbpro/public_client.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index a670b870..f76f5944 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -304,8 +304,9 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): r = self.session.get(url, params=params, auth=self.auth, timeout=30) results = r.json() for result in results: -# if "sleep_interval" in kwargs.keys(): -# time.sleep(kwargs["sleep_interval"]) + #The sleep interval keyword argument was sent. + if "sleep_interval" in kwargs.keys(): + time.sleep(kwargs["sleep_interval"]) if result != "": yield str(result) # If there are no more pages, we're done. Otherwise update `after` @@ -314,13 +315,6 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): # cbpro API doesn't support multiple pages in that case. if not r.headers.get('cb-after') or \ params.get('before') is not None: - #If a sleep_interval was sent, use it - if "sleep_interval" in kwargs.keys(): - print("Sleeping", flush=True) - time.sleep(kwargs["sleep_interval"]) break else: params['after'] = r.headers['cb-after'] - if "sleep_interval" in kwargs.keys(): - print("Sleeping", flush=True) - time.sleep(kwargs["sleep_interval"]) From ebb9ff3b9172a73f8e0d9e9550ffdb998fe9d114 Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Wed, 13 Jan 2021 15:35:47 -0500 Subject: [PATCH 24/31] Maybe the sleep works correctly now. Trying to get the sleep_interval to work correctly --- cbpro/public_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index f76f5944..d0144ebb 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -303,10 +303,10 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): while True: r = self.session.get(url, params=params, auth=self.auth, timeout=30) results = r.json() + #The sleep interval keyword argument was sent. + if "sleep_interval" in kwargs.keys(): + time.sleep(kwargs["sleep_interval"]) for result in results: - #The sleep interval keyword argument was sent. - if "sleep_interval" in kwargs.keys(): - time.sleep(kwargs["sleep_interval"]) if result != "": yield str(result) # If there are no more pages, we're done. Otherwise update `after` From de098e06fb4235f3453066dea4c67c4865b78441 Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Wed, 13 Jan 2021 15:44:43 -0500 Subject: [PATCH 25/31] Moved the cast to a string to the results line Trying to make this work on a Mac from both Python and R --- cbpro/public_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index d0144ebb..470f6807 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -302,13 +302,13 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): while True: r = self.session.get(url, params=params, auth=self.auth, timeout=30) - results = r.json() + results = str(r).json() #The sleep interval keyword argument was sent. if "sleep_interval" in kwargs.keys(): time.sleep(kwargs["sleep_interval"]) for result in results: if result != "": - yield str(result) + yield result # If there are no more pages, we're done. Otherwise update `after` # param to get next page. # If this request included `before` don't get any more pages - the From 216e221545247085e1c1d96e04055f1ecace7121 Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Wed, 13 Jan 2021 15:46:44 -0500 Subject: [PATCH 26/31] Moved the cast to a string again This is the bug that never dies --- cbpro/public_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index 470f6807..a03184d7 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -302,7 +302,7 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): while True: r = self.session.get(url, params=params, auth=self.auth, timeout=30) - results = str(r).json() + results = str(r.json()) #The sleep interval keyword argument was sent. if "sleep_interval" in kwargs.keys(): time.sleep(kwargs["sleep_interval"]) From eaa84295fcb83d03de48908e0b30773d75b7edf5 Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Wed, 13 Jan 2021 15:49:44 -0500 Subject: [PATCH 27/31] Removed the string cast and added an empty yield Not sure if this will help . . . --- cbpro/public_client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index a03184d7..3edc6dd2 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -302,13 +302,15 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): while True: r = self.session.get(url, params=params, auth=self.auth, timeout=30) - results = str(r.json()) + results = r.json() #The sleep interval keyword argument was sent. if "sleep_interval" in kwargs.keys(): time.sleep(kwargs["sleep_interval"]) for result in results: if result != "": yield result + else: + yield "" # If there are no more pages, we're done. Otherwise update `after` # param to get next page. # If this request included `before` don't get any more pages - the From bdc1067723029e5772a0bca524dcc4941962ebd1 Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Wed, 13 Jan 2021 15:53:23 -0500 Subject: [PATCH 28/31] Ugh! --- cbpro/public_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index 3edc6dd2..ec7f2c06 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -310,7 +310,7 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): if result != "": yield result else: - yield "" + yield "{'end':1}" # If there are no more pages, we're done. Otherwise update `after` # param to get next page. # If this request included `before` don't get any more pages - the From 99255990fcf9d44aa1bc3972503edfda62c3ff2e Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Wed, 13 Jan 2021 15:56:23 -0500 Subject: [PATCH 29/31] Back to casting the result to a string At least it works in Python . . . might need to change my R code --- cbpro/public_client.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index ec7f2c06..a784982e 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -308,9 +308,7 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): time.sleep(kwargs["sleep_interval"]) for result in results: if result != "": - yield result - else: - yield "{'end':1}" + yield (str)result # If there are no more pages, we're done. Otherwise update `after` # param to get next page. # If this request included `before` don't get any more pages - the From fdc3e684d4768540a55556172559b06d3e47a75e Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Wed, 13 Jan 2021 16:02:05 -0500 Subject: [PATCH 30/31] Fixing cast to string My syntax is still shaky --- cbpro/public_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index a784982e..d0144ebb 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -308,7 +308,7 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): time.sleep(kwargs["sleep_interval"]) for result in results: if result != "": - yield (str)result + yield str(result) # If there are no more pages, we're done. Otherwise update `after` # param to get next page. # If this request included `before` don't get any more pages - the From 7238b00fc37928e851bf76fb0925257c3255030e Mon Sep 17 00:00:00 2001 From: Angel Aponte Date: Wed, 13 Jan 2021 16:56:32 -0500 Subject: [PATCH 31/31] Removed the cast to a string This is impressively annoying --- cbpro/public_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cbpro/public_client.py b/cbpro/public_client.py index d0144ebb..74b4bbd3 100644 --- a/cbpro/public_client.py +++ b/cbpro/public_client.py @@ -308,7 +308,7 @@ def _send_paginated_message(self, endpoint, params=None, **kwargs): time.sleep(kwargs["sleep_interval"]) for result in results: if result != "": - yield str(result) + yield result # If there are no more pages, we're done. Otherwise update `after` # param to get next page. # If this request included `before` don't get any more pages - the