From 01d89e09a03f719d15a68e1e1b30137175a8f2ae Mon Sep 17 00:00:00 2001 From: RushiT0122 Date: Fri, 3 Nov 2023 10:12:49 +0000 Subject: [PATCH 01/10] add dev-mode changes --- tap_typeform/__init__.py | 6 ++++- tap_typeform/client.py | 53 +++++++++++++++++++++++++++++++++++++--- utils.py | 24 ++++++++++++++++++ 3 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 utils.py diff --git a/tap_typeform/__init__.py b/tap_typeform/__init__.py index 553bec8..7c6dc23 100644 --- a/tap_typeform/__init__.py +++ b/tap_typeform/__init__.py @@ -38,7 +38,11 @@ def validate_form_ids(client, config): def main(): args = utils.parse_args(REQUIRED_CONFIG_KEYS) config = args.config - client = Client(config) + + if args.dev: + LOGGER.warning("Executing Tap in Dev mode",) + + client = Client(config, args.config_path, args.dev) valid_forms = validate_form_ids(client, config) if args.discover: catalog = _discover() diff --git a/tap_typeform/client.py b/tap_typeform/client.py index 13b29dd..1205a22 100644 --- a/tap_typeform/client.py +++ b/tap_typeform/client.py @@ -2,7 +2,10 @@ import backoff import singer +from datetime import timedelta +from singer.utils import now, strftime from requests.exceptions import ChunkedEncodingError, Timeout, ConnectionError +from tap_typeform import utils LOGGER = singer.get_logger() @@ -113,15 +116,23 @@ class Client(object): The client class is used for making REST calls to the Github API. """ BASE_URL = 'https://api.typeform.com' + OAUTH_URL = 'https://api.typeform.com/oauth/token' - def __init__(self, config): - self.token = 'Bearer ' + config.get('token') + def __init__(self, config, config_path, dev_mode): self.metric = config.get('metric') self.session = requests.Session() self.page_size = MAX_RESPONSES_PAGE_SIZE self.form_page_size = FORMS_PAGE_SIZE + self.config_path = config_path self.get_page_size(config) + self.client_id = config.get('client_id') + self.client_secret = config.get('client_secret') + self.refresh_token = config.get('refresh_token') + self.access_token = config.get('token') + self.dev_mode = dev_mode + self.refresh() + # Set and pass request timeout to config param `request_timeout` value. config_request_timeout = config.get('request_timeout') if config_request_timeout and float(config_request_timeout): @@ -129,6 +140,40 @@ def __init__(self, config): else: self.request_timeout = REQUEST_TIMEOUT # If value is 0,"0","" or not passed then it set default to 300 seconds. + def refresh(self): + """ + Checks token expiry and refreshes token if access token is expired + """ + # Existing connections won't have refresh token so use the existing access token + if not self.refresh_token: + return + + # In dev mode, don't refresh access token + if self.dev_mode and self.expires_at: + if not self.access_token: + raise Exception('Access token is missing') + + return + + data = self.post('/oauth/token', + auth_call=True, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + data={'client_id': self.client_id, + 'client_secret': self.client_secret, + 'redirect_uri': self.redirect_uri, + 'refresh_token': self.refresh_token, + 'grant_type': 'refresh_token', + 'scope': 'forms:read accounts:read images:read responses:read themes:read workspaces:read'}) + + self.refresh_token = data['refresh_token'] + self.access_token = data['access_token'] + # pad by 10 seconds for clock drift + self.expires_at = now() + timedelta(seconds=data['expires_in'] - 10) + + utils.write_config(self.config_path, + {"refresh_token": self.refresh_token, + "access_token": self.access_token}) + def get_page_size(self, config): """ This function will get page size from config, @@ -161,8 +206,8 @@ def request(self, url, params={}, **kwargs): if 'headers' not in kwargs: kwargs['headers'] = {} - if self.token: - kwargs['headers']['Authorization'] = self.token + if self.access_token: + kwargs['headers']['Authorization'] = 'Bearer ' + self.access_token LOGGER.info("URL: %s and Params: %s", url, params) response = self.session.get(url, params=params, headers=kwargs['headers'], timeout=self.request_timeout) diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..5e299a8 --- /dev/null +++ b/utils.py @@ -0,0 +1,24 @@ +import json + + +def read_config(config_path): + """ + Performs read on the provided filepath, + returns empty dict if invalid path provided + """ + try: + with open(config_path, 'r') as tap_config: + return json.load(tap_config) + except FileNotFoundError as err: + raise Exception("Failed to load config in dev mode") from err + + +def write_config(config_path, data): + """ + Updates the provided filepath with json format of the `data` object + """ + config = read_config(config_path) + config.update(data) + with open(config_path, 'w') as tap_config: + json.dump(config, tap_config, indent=2) + return config From c6489fef7d79fb88ffe734ebe199689a03787790 Mon Sep 17 00:00:00 2001 From: RushiT0122 Date: Fri, 3 Nov 2023 11:50:24 +0000 Subject: [PATCH 02/10] fix the issues --- setup.py | 2 +- tap_typeform/__init__.py | 6 +++--- tap_typeform/client.py | 36 ++++++++++++++++++------------- utils.py => tap_typeform/utils.py | 0 4 files changed, 25 insertions(+), 19 deletions(-) rename utils.py => tap_typeform/utils.py (100%) diff --git a/setup.py b/setup.py index c9e3d56..c9a2642 100755 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ url="http://singer.io", classifiers=["Programming Language :: Python :: 3 :: Only"], install_requires=[ - "singer-python==5.10.0", + "singer-python==5.13.0", "pendulum", "ratelimit", "backoff", diff --git a/tap_typeform/__init__.py b/tap_typeform/__init__.py index 7c6dc23..f9f08e8 100644 --- a/tap_typeform/__init__.py +++ b/tap_typeform/__init__.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 import singer -from singer import utils +from singer import utils as _utils from tap_typeform.discover import discover as _discover from tap_typeform.sync import sync as _sync from tap_typeform.client import Client @@ -34,9 +34,9 @@ def validate_form_ids(client, config): return config_forms -@utils.handle_top_exception(LOGGER) +@_utils.handle_top_exception(LOGGER) def main(): - args = utils.parse_args(REQUIRED_CONFIG_KEYS) + args = _utils.parse_args(REQUIRED_CONFIG_KEYS) config = args.config if args.dev: diff --git a/tap_typeform/client.py b/tap_typeform/client.py index 1205a22..965cdde 100644 --- a/tap_typeform/client.py +++ b/tap_typeform/client.py @@ -3,9 +3,10 @@ import singer from datetime import timedelta -from singer.utils import now, strftime +from singer.utils import now from requests.exceptions import ChunkedEncodingError, Timeout, ConnectionError -from tap_typeform import utils +from tap_typeform.utils import write_config + LOGGER = singer.get_logger() @@ -140,6 +141,10 @@ def __init__(self, config, config_path, dev_mode): else: self.request_timeout = REQUEST_TIMEOUT # If value is 0,"0","" or not passed then it set default to 300 seconds. + @backoff.on_exception(backoff.expo,(Timeout, ConnectionError), # Backoff for Timeout and ConnectionError. + max_tries=5, factor=2, jitter=None) + @backoff.on_exception(backoff.expo, (TypeformInternalError, TypeformNotAvailableError, TypeformTooManyError, ChunkedEncodingError), + max_tries=3, factor=2) def refresh(self): """ Checks token expiry and refreshes token if access token is expired @@ -149,28 +154,29 @@ def refresh(self): return # In dev mode, don't refresh access token - if self.dev_mode and self.expires_at: + if self.dev_mode: if not self.access_token: raise Exception('Access token is missing') return - data = self.post('/oauth/token', - auth_call=True, - headers={"Content-Type": "application/x-www-form-urlencoded"}, - data={'client_id': self.client_id, - 'client_secret': self.client_secret, - 'redirect_uri': self.redirect_uri, - 'refresh_token': self.refresh_token, - 'grant_type': 'refresh_token', - 'scope': 'forms:read accounts:read images:read responses:read themes:read workspaces:read'}) + response = self.session.request("POST", + url = self.OAUTH_URL, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + data={'client_id': self.client_id, + 'client_secret': self.client_secret, + 'refresh_token': self.refresh_token, + 'grant_type': 'refresh_token', + 'scope': 'forms:read accounts:read images:read responses:read themes:read workspaces:read'}) + + if response.status_code != 200: + raise_for_error(response) + data = response.json() self.refresh_token = data['refresh_token'] self.access_token = data['access_token'] - # pad by 10 seconds for clock drift - self.expires_at = now() + timedelta(seconds=data['expires_in'] - 10) - utils.write_config(self.config_path, + write_config(self.config_path, {"refresh_token": self.refresh_token, "access_token": self.access_token}) diff --git a/utils.py b/tap_typeform/utils.py similarity index 100% rename from utils.py rename to tap_typeform/utils.py From dd5ac0bdd21f447380e47051b52ea02ac3e65cd9 Mon Sep 17 00:00:00 2001 From: RushiT0122 Date: Fri, 3 Nov 2023 12:22:17 +0000 Subject: [PATCH 03/10] fix exception handling and page size unit tests --- tests/unittests/test_exception_handling.py | 39 ++++++++++++++++++---- tests/unittests/test_page_size.py | 37 +++++++++++++++----- 2 files changed, 60 insertions(+), 16 deletions(-) diff --git a/tests/unittests/test_exception_handling.py b/tests/unittests/test_exception_handling.py index 14361c1..55b8f09 100644 --- a/tests/unittests/test_exception_handling.py +++ b/tests/unittests/test_exception_handling.py @@ -1,3 +1,5 @@ +import json +import os import unittest from unittest import mock from parameterized import parameterized @@ -6,6 +8,20 @@ from tap_typeform.client import ERROR_CODE_EXCEPTION_MAPPING import tap_typeform.client as client_ + +test_config = { + "client_id": "client_id", + "client_secret": "client_secret", + "access_token": "old_access_token" +} +test_config_path = "/tmp/test_config.json" + +def write_new_config_file(): + with open(test_config_path, 'w') as config: + # Reset tokens while writing the test config + test_config["access_token"] = "old_access_token" + config.write(json.dumps(test_config)) + class Mockresponse: def __init__(self, resp, status_code, content=[], headers=None, raise_error=False): self.json_data = resp @@ -86,6 +102,10 @@ class TestClientErrorHandling(unittest.TestCase): endpoint = "forms" + def tearDown(self): + if os.path.isfile(test_config_path): + os.remove(test_config_path) + @parameterized.expand([ (client_.TypeformBadRequestError, mocked_badrequest_400_error, 400), (client_.TypeformUnauthorizedError, mocked_unauthorized_401_error, 401), @@ -100,16 +120,16 @@ def test_error_handling(self, mock_session, mock_sleep, error, mock_response, er """ Test error is raised with an expected error message. """ - config = {'token': '123'} - client = client_.Client(config) + write_new_config_file() + client = client_.Client(test_config, test_config_path, False) url = client.build_url(self.endpoint) mock_session.side_effect=mock_response error_message = ERROR_CODE_EXCEPTION_MAPPING.get(err_code, {}).get("message", "") - + expected_error_message = "HTTP-error-code: {}, Error: {}".format(err_code, error_message) with self.assertRaises(error) as e: client.request(url) - + # Verifying the message formed for the custom exception self.assertEqual(str(e.exception), expected_error_message) @@ -119,7 +139,8 @@ def test_success_response(self, mock_logger, mock_raise_error, mock_session, moc """ Test that for success response, error is not raised """ - client = client_.Client({'token': '123'}) + write_new_config_file() + client = client_.Client(test_config, test_config_path, False) mock_session.return_value=get_mock_http_response(200, '{"total_items": 10}') client.request("") @@ -138,6 +159,10 @@ class TestClientBackoffHandling(unittest.TestCase): endpoint = "forms" + def tearDown(self): + if os.path.isfile(test_config_path): + os.remove(test_config_path) + @parameterized.expand([ (requests.exceptions.ConnectionError, requests.exceptions.ConnectionError, 5), (requests.exceptions.Timeout, requests.exceptions.Timeout, 5), @@ -151,8 +176,8 @@ def test_back_off_error_handling(self, mock_session, mock_sleep, error,mock_resp Test handling of backoff that function is retrying expected times """ mock_session.side_effect = mock_response - config = {'token': '123'} - client = client_.Client(config) + write_new_config_file() + client = client_.Client(test_config, test_config_path, False) url = client.build_url(self.endpoint) with self.assertRaises(error): client.request(url) diff --git a/tests/unittests/test_page_size.py b/tests/unittests/test_page_size.py index 33b2c6f..b089a57 100644 --- a/tests/unittests/test_page_size.py +++ b/tests/unittests/test_page_size.py @@ -1,3 +1,5 @@ +import json +import os import unittest from unittest.mock import patch from parameterized import parameterized @@ -15,6 +17,19 @@ PAGE_SIZE_DEFAULT_FORMS = 200 PAGE_SIZE_DEFAULT = 1000 +test_config = { + "client_id": "client_id", + "client_secret": "client_secret", + "access_token": "old_access_token" +} +test_config_path = "/tmp/test_config.json" + +def write_new_config_file(): + with open(test_config_path, 'w') as config: + # Reset tokens while writing the test config + test_config["access_token"] = "old_access_token" + config.write(json.dumps(test_config)) + # Mock response object def get_mock_http_response(*args, **kwargs): contents = '{"accounts":[{"id": 12}]}' @@ -38,9 +53,10 @@ def test_page_size_for_diff_values_forms(self, page_size_value, expected_value): - For null string, zero(string), zero(integer), takes default integer value """ self.endpoint = "forms" - config = {'token': '123', "page_size": page_size_value} - client = client_.Client(config) - client.get_page_size(config) + write_new_config_file() + client = client_.Client(test_config, test_config_path, False) + test_config["page_size"] = page_size_value + client.get_page_size(test_config) # Verify the form_page_size is the same as the expected value self.assertEqual(client.form_page_size, expected_value) @@ -57,9 +73,10 @@ def test_page_size_for_diff_values(self, page_size_value, expected_value): - For null string, zero(string), zero(integer), takes default integer value """ self.endpoint = "landings" - config = {'token': '123', "page_size": page_size_value} - client = client_.Client(config) - client.get_page_size(config) + write_new_config_file() + client = client_.Client(test_config, test_config_path, False) + test_config["page_size"] = page_size_value + client.get_page_size(test_config) # Verify the page_size is the same as the expected value self.assertEqual(client.page_size, expected_value) @@ -77,10 +94,12 @@ def test_page_size_for_invalid_values(self, page_size_value, error): """ self.endpoint = "landings" print(page_size_value) - config = {'token': '123', "page_size": page_size_value} + test_config = {'token': '123', "page_size": page_size_value} # Verify the tap raises Exception with self.assertRaises(error) as e: - client = client_.Client(config) - client.get_page_size(config) + write_new_config_file() + client = client_.Client(test_config, test_config_path, False) + test_config["page_size"] = page_size_value + client.get_page_size(test_config) # Verify the tap raises an error with expected error message self.assertEqual(str(e.exception), "The entered page size is invalid, it should be a valid integer.") From e227c1007c63342d0ef8562910cc8f7439719b5a Mon Sep 17 00:00:00 2001 From: RushiT0122 Date: Mon, 6 Nov 2023 13:38:47 +0000 Subject: [PATCH 04/10] minor tap and refactoring changes --- tap_typeform/__init__.py | 2 +- tap_typeform/client.py | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/tap_typeform/__init__.py b/tap_typeform/__init__.py index f9f08e8..fccfc93 100644 --- a/tap_typeform/__init__.py +++ b/tap_typeform/__init__.py @@ -40,7 +40,7 @@ def main(): config = args.config if args.dev: - LOGGER.warning("Executing Tap in Dev mode",) + LOGGER.warning("Executing Tap in Dev mode") client = Client(config, args.config_path, args.dev) valid_forms = validate_form_ids(client, config) diff --git a/tap_typeform/client.py b/tap_typeform/client.py index 965cdde..5d76b11 100644 --- a/tap_typeform/client.py +++ b/tap_typeform/client.py @@ -141,7 +141,7 @@ def __init__(self, config, config_path, dev_mode): else: self.request_timeout = REQUEST_TIMEOUT # If value is 0,"0","" or not passed then it set default to 300 seconds. - @backoff.on_exception(backoff.expo,(Timeout, ConnectionError), # Backoff for Timeout and ConnectionError. + @backoff.on_exception(backoff.expo,(Timeout, ConnectionError), # Backoff for Timeout and ConnectionError. max_tries=5, factor=2, jitter=None) @backoff.on_exception(backoff.expo, (TypeformInternalError, TypeformNotAvailableError, TypeformTooManyError, ChunkedEncodingError), max_tries=3, factor=2) @@ -160,14 +160,14 @@ def refresh(self): return - response = self.session.request("POST", - url = self.OAUTH_URL, - headers={"Content-Type": "application/x-www-form-urlencoded"}, - data={'client_id': self.client_id, - 'client_secret': self.client_secret, - 'refresh_token': self.refresh_token, - 'grant_type': 'refresh_token', - 'scope': 'forms:read accounts:read images:read responses:read themes:read workspaces:read'}) + response = self.session.post(url=self.OAUTH_URL, + headers={ + "Content-Type": "application/x-www-form-urlencoded"}, + data={'client_id': self.client_id, + 'client_secret': self.client_secret, + 'refresh_token': self.refresh_token, + 'grant_type': 'refresh_token', + 'scope': 'forms:read accounts:read images:read responses:read themes:read workspaces:read'}) if response.status_code != 200: raise_for_error(response) @@ -177,8 +177,8 @@ def refresh(self): self.access_token = data['access_token'] write_config(self.config_path, - {"refresh_token": self.refresh_token, - "access_token": self.access_token}) + {"refresh_token": self.refresh_token, + "access_token": self.access_token}) def get_page_size(self, config): """ @@ -189,7 +189,7 @@ def get_page_size(self, config): if page_size is None: return if ((type(page_size) == int or type(page_size) == float) and (page_size > 0)) or \ - (type(page_size) == str and page_size.replace('.', '', 1).isdigit() and (float(page_size) > 0) ): + (type(page_size) == str and page_size.replace('.', '', 1).isdigit() and (float(page_size) > 0)): self.page_size = int(float(page_size)) self.form_page_size = min(self.form_page_size, self.page_size) else: @@ -201,10 +201,10 @@ def build_url(self, endpoint): """ return f"{self.BASE_URL}/{endpoint}" - @backoff.on_exception(backoff.expo,(Timeout, ConnectionError), # Backoff for Timeout and ConnectionError. - max_tries=5, factor=2, jitter=None) + @backoff.on_exception(backoff.expo, (Timeout, ConnectionError), # Backoff for Timeout and ConnectionError. + max_tries=5, factor=2, jitter=None) @backoff.on_exception(backoff.expo, (TypeformInternalError, TypeformNotAvailableError, TypeformTooManyError, ChunkedEncodingError), - max_tries=3, factor=2) + max_tries=3, factor=2) def request(self, url, params={}, **kwargs): """ Call rest API and return the response in case of status code 200. From 16f7ace901f4cc3b03667309c4fa37f3cb6e479e Mon Sep 17 00:00:00 2001 From: RushiT0122 Date: Mon, 6 Nov 2023 14:00:29 +0000 Subject: [PATCH 05/10] update unit tests --- tests/unittests/test_dev_mode.py | 97 ++++++++++++++++++++++ tests/unittests/test_exception_handling.py | 57 +++++++++---- tests/unittests/test_main.py | 28 ++++++- tests/unittests/test_request_timeout.py | 29 +++++-- tests/unittests/test_sync_obj.py | 32 +++++-- 5 files changed, 211 insertions(+), 32 deletions(-) create mode 100644 tests/unittests/test_dev_mode.py diff --git a/tests/unittests/test_dev_mode.py b/tests/unittests/test_dev_mode.py new file mode 100644 index 0000000..9198843 --- /dev/null +++ b/tests/unittests/test_dev_mode.py @@ -0,0 +1,97 @@ +import unittest +import os +import json +import requests +from unittest import mock + +from tap_typeform.client import Client + + +http_response = {"refresh_token": "new_refresh_token", + "access_token": "new_access_token"} + +test_config_path = "/tmp/test_config.json" + + +class Mockresponse: + """ Mock response object class.""" + + def __init__(self, status_code, raise_error, text=""): + self.status_code = status_code + self.raise_error = raise_error + self.text = text + + def raise_for_status(self): + if not self.raise_error: + return self.status_code + + raise requests.HTTPError("Sample message") + + def json(self): + """ Response JSON method.""" + return self.text + + +def get_mock_http_response(status_code): + """Return http mock response.""" + response = requests.Response() + response.status_code = status_code + return response + + +def get_response(status_code, raise_error=True, text=""): + """ Returns required mock response. """ + return Mockresponse(status_code, raise_error=raise_error, text=text) + + +def write_new_config_file(**kwargs): + test_config = {} + with open(test_config_path, 'w') as config: + for key, value in kwargs.items(): + test_config[key] = value + config.write(json.dumps(test_config)) + + +class TestDevMode(unittest.TestCase): + def tearDown(self): + if os.path.isfile(test_config_path): + os.remove(test_config_path) + + @mock.patch("requests.Session.request") + def test_dev_mode_not_enabled(self, mock_post_request): + test_config = {"refresh_token": "old_refresh_token", + "token": "old_access_token"} + write_new_config_file(**test_config) + mock_post_request.side_effect = [get_response(200, raise_error=False, text=http_response)] + client = Client(config=test_config, config_path=test_config_path, dev_mode=False) + self.assertEqual(client.refresh_token, "new_refresh_token") + self.assertEqual(client.access_token, "new_access_token") + + @mock.patch("requests.Session.request") + def test_dev_mode_enabled(self, mock_post_request): + test_config = {"refresh_token": "old_refresh_token", + "token": "old_access_token"} + write_new_config_file(**test_config) + mock_post_request.side_effect = [get_response(200, raise_error=False, text=http_response)] + client = Client(config=test_config, config_path=test_config_path, dev_mode=True) + self.assertEqual(client.refresh_token, "old_refresh_token") + self.assertEqual(client.access_token, "old_access_token") + + + @mock.patch("requests.Session.request") + def test_no_refresh_token_not_dev_mode_enabled(self, mock_post_request): + test_config = {"token": "old_access_token"} + write_new_config_file(**test_config) + mock_post_request.side_effect = [get_response(200, raise_error=False, text=http_response)] + client = Client(config=test_config, config_path=test_config_path, dev_mode=False) + self.assertIsNone(client.refresh_token) + self.assertEqual(client.access_token, "old_access_token") + + @mock.patch("requests.Session.request") + def test_no_refresh_token_dev_mode_enabled(self, mock_post_request): + test_config = {"token": "old_access_token"} + write_new_config_file(**test_config) + mock_post_request.side_effect = [get_response(200, raise_error=False, text=http_response)] + client = Client(config=test_config, config_path=test_config_path, dev_mode=True) + self.assertIsNone(client.refresh_token) + self.assertEqual(client.access_token, "old_access_token") diff --git a/tests/unittests/test_exception_handling.py b/tests/unittests/test_exception_handling.py index 55b8f09..69bc5ba 100644 --- a/tests/unittests/test_exception_handling.py +++ b/tests/unittests/test_exception_handling.py @@ -9,17 +9,15 @@ import tap_typeform.client as client_ -test_config = { - "client_id": "client_id", - "client_secret": "client_secret", - "access_token": "old_access_token" -} + test_config_path = "/tmp/test_config.json" -def write_new_config_file(): + +def write_new_config_file(**kwargs): + test_config = {} with open(test_config_path, 'w') as config: - # Reset tokens while writing the test config - test_config["access_token"] = "old_access_token" + for key, value in kwargs.items(): + test_config[key] = value config.write(json.dumps(test_config)) class Mockresponse: @@ -120,7 +118,8 @@ def test_error_handling(self, mock_session, mock_sleep, error, mock_response, er """ Test error is raised with an expected error message. """ - write_new_config_file() + test_config = {"token": ""} + write_new_config_file(**test_config) client = client_.Client(test_config, test_config_path, False) url = client.build_url(self.endpoint) mock_session.side_effect=mock_response @@ -139,7 +138,8 @@ def test_success_response(self, mock_logger, mock_raise_error, mock_session, moc """ Test that for success response, error is not raised """ - write_new_config_file() + test_config = {"token": ""} + write_new_config_file(**test_config) client = client_.Client(test_config, test_config_path, False) mock_session.return_value=get_mock_http_response(200, '{"total_items": 10}') client.request("") @@ -150,8 +150,7 @@ def test_success_response(self, mock_logger, mock_raise_error, mock_session, moc # Verify `raw data item` logger is called mock_logger.assert_called_with("raw data items= 10") -@mock.patch("time.sleep") -@mock.patch('tap_typeform.client.requests.Session.get') + class TestClientBackoffHandling(unittest.TestCase): """ Test handling of backoff for Timeout, ConnectionError, ChunkEncoding, 5xx, 429 errors. @@ -171,16 +170,42 @@ def tearDown(self): (client_.TypeformNotAvailableError, mocked_not_available_503_error, 3), (client_.TypeformTooManyError, mocked_failed_429_request, 3), ]) - def test_back_off_error_handling(self, mock_session, mock_sleep, error,mock_response, expected_call_count): + @mock.patch("time.sleep") + @mock.patch('tap_typeform.client.requests.Session.get') + def test_back_off_error_handling(self, error,mock_response, expected_call_count, mock_session_get, mock_sleep): """ Test handling of backoff that function is retrying expected times """ - mock_session.side_effect = mock_response - write_new_config_file() + mock_session_get.side_effect = mock_response + test_config = {"token": ""} + write_new_config_file(**test_config) client = client_.Client(test_config, test_config_path, False) url = client.build_url(self.endpoint) with self.assertRaises(error): client.request(url) # Verify `client.requests` backoff expected times - self.assertEqual(mock_session.call_count, expected_call_count) + self.assertEqual(mock_session_get.call_count, expected_call_count) + + @parameterized.expand([ + (requests.exceptions.ConnectionError, requests.exceptions.ConnectionError, 5), + (requests.exceptions.Timeout, requests.exceptions.Timeout, 5), + (requests.exceptions.ChunkedEncodingError, requests.exceptions.ChunkedEncodingError, 3), + (client_.TypeformInternalError, mocked_internalservererror_500_error, 3), + (client_.TypeformNotAvailableError, mocked_not_available_503_error, 3), + (client_.TypeformTooManyError, mocked_failed_429_request, 3), + ]) + @mock.patch("time.sleep") + @mock.patch('tap_typeform.client.requests.Session.post') + def test_refresh_token_back_off_error_handling(self, error, mock_response, expected_call_count, mock_session_post, mock_sleep): + """ + Test handling of backoff that function is retrying expected times + """ + mock_session_post.side_effect = mock_response + test_config = {"token": "access_token", "refresh_token": "refresh_token"} + write_new_config_file(**test_config) + with self.assertRaises(error): + client_.Client(test_config, test_config_path, False) + + # Verify `client.requests` backoff expected times + self.assertEqual(mock_session_post.call_count, expected_call_count) diff --git a/tests/unittests/test_main.py b/tests/unittests/test_main.py index d2dde1a..0c19929 100644 --- a/tests/unittests/test_main.py +++ b/tests/unittests/test_main.py @@ -8,11 +8,13 @@ class MockArgs: """Mock args object class""" - def __init__(self, config=None, catalog=None, state={}, discover=False) -> None: + def __init__(self, config=None, catalog=None, state={}, discover=False, dev=False) -> None: self.config = config self.catalog = catalog self.state = state self.discover = discover + self.dev = dev + self.config_path = "" @mock.patch("tap_typeform.validate_form_ids") @@ -82,6 +84,30 @@ def test_sync_with_state(self, mock_sync, mock_discover, mock_args, mock_validat # Verify `_sync` is called with expected arguments mock_sync.assert_called_with(mock.ANY, self.mock_config, mock_state, self.mock_catalog, mock_validate.return_value) + def test_discover_with_dev_mode_enabled(self, mock_sync, mock_discover, mock_args, mock_validate): + """ + Test `_discover` function is called for discover mode. + """ + mock_discover.dump.return_value = dict() + mock_args.return_value = MockArgs(discover=True, config=self.mock_config, dev=True) + main() + + self.assertTrue(mock_discover.called) + self.assertFalse(mock_sync.called) + + def test_sync_with_dev_mode_enabled(self, mock_sync, mock_discover, mock_args, mock_validate): + """ + Test `_discover` function is called for discover mode. + """ + mock_discover.dump.return_value = dict() + mock_args.return_value = MockArgs(config=self.mock_config, + catalog=Catalog.from_dict(self.mock_catalog), + dev=True) + main() + + self.assertFalse(mock_discover.called) + self.assertTrue(mock_sync.called) + @mock.patch("tap_typeform.Forms") class TestValidateFormIds(unittest.TestCase): diff --git a/tests/unittests/test_request_timeout.py b/tests/unittests/test_request_timeout.py index 5843452..d441b39 100644 --- a/tests/unittests/test_request_timeout.py +++ b/tests/unittests/test_request_timeout.py @@ -1,3 +1,4 @@ +import json import unittest from unittest.mock import patch from parameterized import parameterized @@ -9,6 +10,17 @@ REQUEST_TIMEOUT_STR = "300" REQUEST_TIMEOUT_FLOAT = 300.0 + +test_config_path = "/tmp/test_config.json" + +def write_new_config_file(**kwargs): + test_config = {} + with open(test_config_path, 'w') as config: + for key, value in kwargs.items(): + test_config[key] = value + config.write(json.dumps(test_config)) + + # Mock response object def get_mock_http_response(*args, **kwargs): contents = '{"accounts":[{"id": 12}]}' @@ -23,7 +35,7 @@ def get_mock_http_response(*args, **kwargs): class TestRequestTimeoutsValue(unittest.TestCase): endpoint = "forms" - + @parameterized.expand([ (REQUEST_TIMEOUT_STR, REQUEST_TIMEOUT_FLOAT), (REQUEST_TIMEOUT_INT, REQUEST_TIMEOUT_FLOAT), @@ -38,10 +50,12 @@ def test_request_timeout_for_diff_values(self, mocked_request, mock_get, mock_sl - For string, integer, float type of values, converts to float - For null string, zero(string), zero(integer), takes default integer value """ - config = {'token': '123', "request_timeout": time_out_value} - client = client_.Client(config) + test_config = {"token": "old_access_token", + "request_timeout": time_out_value} + write_new_config_file(**test_config) + client = client_.Client(test_config, test_config_path, False) url = client.build_url(self.endpoint) - + # Call request method which calls `requests.Session.get` with timeout client.request(url) @@ -52,9 +66,10 @@ def test_request_timeout_for_diff_values(self, mocked_request, mock_get, mock_sl def test_no_request_timeout_value_in_config(self, mocked_request, mock_send, mock_sleep): """ Verify that if request_timeout is not provided in the config then the default value is used - """ - config = {'token': '123'} - client = client_.Client(config) + """ + test_config = {"token": "old_access_token"} + write_new_config_file(**test_config) + client = client_.Client(test_config, test_config_path, False) url = client.build_url(self.endpoint) # Call request method which calls `requests.Session.get` with timeout client.request(url) diff --git a/tests/unittests/test_sync_obj.py b/tests/unittests/test_sync_obj.py index 9bf88d2..0f58fae 100644 --- a/tests/unittests/test_sync_obj.py +++ b/tests/unittests/test_sync_obj.py @@ -1,9 +1,22 @@ +import json import unittest from unittest import mock from parameterized import parameterized from tap_typeform.client import Client from tap_typeform.streams import Forms, SubmittedLandings, Questions, Answers, UnsubmittedLandings +test_config = {"token": ""} +test_config_path = "/tmp/test_config.json" + + +def write_new_config_file(**kwargs): + test_config = {} + with open(test_config_path, 'w') as config: + for key, value in kwargs.items(): + test_config[key] = value + config.write(json.dumps(test_config)) + + def get_stream_catalog(stream_name, selected = False): """ Returns catalog for each stream. @@ -41,13 +54,14 @@ def test_sync_object(self, mock_request, mock_write_records): """ Test `sync_obj` for full table streams. """ - client = Client({"token": ""}) + write_new_config_file(**test_config) + client = Client(test_config, test_config_path, False) test_stream = Questions() expected_records = [ {"id": 1, "question_id": 1, "form_id": "form1"}, {"id": 2, "question_id": 2, "form_id": "form1"}, ] - + records = [ {"id": 1}, {"id": 2}, @@ -72,13 +86,14 @@ class TestIncrementalStream(unittest.TestCase): """ Test incremental streams methods. """ - + @mock.patch("tap_typeform.streams.IncrementalStream.write_records") def test_sync_obj(self, mock_write_records, mock_add_field, mock_request): """ - Test `sync_obj` method of incremental streams. + Test `sync_obj` method of incremental streams. """ - client = Client({"token": ""}) + write_new_config_file(**test_config) + client = Client(test_config, test_config_path, False) test_stream = SubmittedLandings() records = [ @@ -169,7 +184,7 @@ class TestFormsStream(unittest.TestCase): """ Test `sync_obj` method for Forms stream. """ - + @mock.patch("tap_typeform.streams.IncrementalStream.write_records") def test_sync_obj(self, mock_write_records, mock_requests): mock_write_records.return_value = "" @@ -179,9 +194,10 @@ def test_sync_obj(self, mock_write_records, mock_requests): {"items": [], "page_count": 3}, {"items": [], "page_count": 3}, ] - client = Client({"token": ""}) + write_new_config_file(**test_config) + client = Client(test_config, test_config_path, False) test_stream = Forms() - + test_stream.sync_obj(client, {}, catalogs, "", ['forms'], {'forms': 0}) # Verify that write records called 3 time From 9652fa2358af2b732bc533ffa700af7f206488a7 Mon Sep 17 00:00:00 2001 From: RushiT0122 Date: Mon, 6 Nov 2023 15:38:56 +0000 Subject: [PATCH 06/10] refactoring chaanges --- tap_typeform/client.py | 8 +++--- tap_typeform/utils.py | 4 +-- tests/unittests/test_dev_mode.py | 17 ++++++------ tests/unittests/test_exception_handling.py | 6 ++--- tests/unittests/test_page_size.py | 30 ++++++++++------------ tests/unittests/test_request_timeout.py | 2 +- 6 files changed, 33 insertions(+), 34 deletions(-) diff --git a/tap_typeform/client.py b/tap_typeform/client.py index 5d76b11..b5b78aa 100644 --- a/tap_typeform/client.py +++ b/tap_typeform/client.py @@ -147,7 +147,7 @@ def __init__(self, config, config_path, dev_mode): max_tries=3, factor=2) def refresh(self): """ - Checks token expiry and refreshes token if access token is expired + Refreshes access token and refresh token """ # Existing connections won't have refresh token so use the existing access token if not self.refresh_token: @@ -162,7 +162,7 @@ def refresh(self): response = self.session.post(url=self.OAUTH_URL, headers={ - "Content-Type": "application/x-www-form-urlencoded"}, + 'Content-Type": "application/x-www-form-urlencoded'}, data={'client_id': self.client_id, 'client_secret': self.client_secret, 'refresh_token': self.refresh_token, @@ -177,8 +177,8 @@ def refresh(self): self.access_token = data['access_token'] write_config(self.config_path, - {"refresh_token": self.refresh_token, - "access_token": self.access_token}) + {'refresh_token': self.refresh_token, + 'access_token': self.access_token}) def get_page_size(self, config): """ diff --git a/tap_typeform/utils.py b/tap_typeform/utils.py index 5e299a8..e828134 100644 --- a/tap_typeform/utils.py +++ b/tap_typeform/utils.py @@ -7,7 +7,7 @@ def read_config(config_path): returns empty dict if invalid path provided """ try: - with open(config_path, 'r') as tap_config: + with open(config_path, "r") as tap_config: return json.load(tap_config) except FileNotFoundError as err: raise Exception("Failed to load config in dev mode") from err @@ -19,6 +19,6 @@ def write_config(config_path, data): """ config = read_config(config_path) config.update(data) - with open(config_path, 'w') as tap_config: + with open(config_path, "w") as tap_config: json.dump(config, tap_config, indent=2) return config diff --git a/tests/unittests/test_dev_mode.py b/tests/unittests/test_dev_mode.py index 9198843..50035d6 100644 --- a/tests/unittests/test_dev_mode.py +++ b/tests/unittests/test_dev_mode.py @@ -13,6 +13,15 @@ test_config_path = "/tmp/test_config.json" + +def write_new_config_file(**kwargs): + test_config = {} + with open(test_config_path, "w") as config: + for key, value in kwargs.items(): + test_config[key] = value + config.write(json.dumps(test_config)) + + class Mockresponse: """ Mock response object class.""" @@ -44,14 +53,6 @@ def get_response(status_code, raise_error=True, text=""): return Mockresponse(status_code, raise_error=raise_error, text=text) -def write_new_config_file(**kwargs): - test_config = {} - with open(test_config_path, 'w') as config: - for key, value in kwargs.items(): - test_config[key] = value - config.write(json.dumps(test_config)) - - class TestDevMode(unittest.TestCase): def tearDown(self): if os.path.isfile(test_config_path): diff --git a/tests/unittests/test_exception_handling.py b/tests/unittests/test_exception_handling.py index 69bc5ba..80bafaf 100644 --- a/tests/unittests/test_exception_handling.py +++ b/tests/unittests/test_exception_handling.py @@ -15,7 +15,7 @@ def write_new_config_file(**kwargs): test_config = {} - with open(test_config_path, 'w') as config: + with open(test_config_path, "w") as config: for key, value in kwargs.items(): test_config[key] = value config.write(json.dumps(test_config)) @@ -171,7 +171,7 @@ def tearDown(self): (client_.TypeformTooManyError, mocked_failed_429_request, 3), ]) @mock.patch("time.sleep") - @mock.patch('tap_typeform.client.requests.Session.get') + @mock.patch("tap_typeform.client.requests.Session.get") def test_back_off_error_handling(self, error,mock_response, expected_call_count, mock_session_get, mock_sleep): """ Test handling of backoff that function is retrying expected times @@ -196,7 +196,7 @@ def test_back_off_error_handling(self, error,mock_response, expected_call_count, (client_.TypeformTooManyError, mocked_failed_429_request, 3), ]) @mock.patch("time.sleep") - @mock.patch('tap_typeform.client.requests.Session.post') + @mock.patch("tap_typeform.client.requests.Session.post") def test_refresh_token_back_off_error_handling(self, error, mock_response, expected_call_count, mock_session_post, mock_sleep): """ Test handling of backoff that function is retrying expected times diff --git a/tests/unittests/test_page_size.py b/tests/unittests/test_page_size.py index b089a57..6857cd9 100644 --- a/tests/unittests/test_page_size.py +++ b/tests/unittests/test_page_size.py @@ -17,19 +17,18 @@ PAGE_SIZE_DEFAULT_FORMS = 200 PAGE_SIZE_DEFAULT = 1000 -test_config = { - "client_id": "client_id", - "client_secret": "client_secret", - "access_token": "old_access_token" -} + test_config_path = "/tmp/test_config.json" -def write_new_config_file(): - with open(test_config_path, 'w') as config: - # Reset tokens while writing the test config - test_config["access_token"] = "old_access_token" + +def write_new_config_file(**kwargs): + test_config = {} + with open(test_config_path, "w") as config: + for key, value in kwargs.items(): + test_config[key] = value config.write(json.dumps(test_config)) + # Mock response object def get_mock_http_response(*args, **kwargs): contents = '{"accounts":[{"id": 12}]}' @@ -53,9 +52,9 @@ def test_page_size_for_diff_values_forms(self, page_size_value, expected_value): - For null string, zero(string), zero(integer), takes default integer value """ self.endpoint = "forms" - write_new_config_file() + test_config = {"token": "access_token", "page_size": page_size_value} + write_new_config_file(**test_config) client = client_.Client(test_config, test_config_path, False) - test_config["page_size"] = page_size_value client.get_page_size(test_config) # Verify the form_page_size is the same as the expected value @@ -73,9 +72,9 @@ def test_page_size_for_diff_values(self, page_size_value, expected_value): - For null string, zero(string), zero(integer), takes default integer value """ self.endpoint = "landings" - write_new_config_file() + test_config = {"token": "access_token", "page_size": page_size_value} + write_new_config_file(**test_config) client = client_.Client(test_config, test_config_path, False) - test_config["page_size"] = page_size_value client.get_page_size(test_config) # Verify the page_size is the same as the expected value @@ -94,12 +93,11 @@ def test_page_size_for_invalid_values(self, page_size_value, error): """ self.endpoint = "landings" print(page_size_value) - test_config = {'token': '123', "page_size": page_size_value} + test_config = {"token": "access_token", "page_size": page_size_value} # Verify the tap raises Exception with self.assertRaises(error) as e: - write_new_config_file() + write_new_config_file(**test_config) client = client_.Client(test_config, test_config_path, False) - test_config["page_size"] = page_size_value client.get_page_size(test_config) # Verify the tap raises an error with expected error message self.assertEqual(str(e.exception), "The entered page size is invalid, it should be a valid integer.") diff --git a/tests/unittests/test_request_timeout.py b/tests/unittests/test_request_timeout.py index d441b39..2158cb9 100644 --- a/tests/unittests/test_request_timeout.py +++ b/tests/unittests/test_request_timeout.py @@ -15,7 +15,7 @@ def write_new_config_file(**kwargs): test_config = {} - with open(test_config_path, 'w') as config: + with open(test_config_path, "w") as config: for key, value in kwargs.items(): test_config[key] = value config.write(json.dumps(test_config)) From 85a0fb5ef0ecf314d0cd018df97ab327e4583d6d Mon Sep 17 00:00:00 2001 From: RushiT0122 Date: Mon, 6 Nov 2023 16:16:08 +0000 Subject: [PATCH 07/10] fix the post call --- tap_typeform/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tap_typeform/client.py b/tap_typeform/client.py index b5b78aa..8181382 100644 --- a/tap_typeform/client.py +++ b/tap_typeform/client.py @@ -162,7 +162,7 @@ def refresh(self): response = self.session.post(url=self.OAUTH_URL, headers={ - 'Content-Type": "application/x-www-form-urlencoded'}, + 'Content-Type': 'application/x-www-form-urlencoded'}, data={'client_id': self.client_id, 'client_secret': self.client_secret, 'refresh_token': self.refresh_token, @@ -178,7 +178,7 @@ def refresh(self): write_config(self.config_path, {'refresh_token': self.refresh_token, - 'access_token': self.access_token}) + 'token': self.access_token}) def get_page_size(self, config): """ From 5db8194c486d31baaf04e5276771a2a5c852d873 Mon Sep 17 00:00:00 2001 From: RushiT0122 Date: Mon, 6 Nov 2023 18:36:30 +0000 Subject: [PATCH 08/10] update sample config.json --- example.config.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/example.config.json b/example.config.json index 7527077..4ed715f 100644 --- a/example.config.json +++ b/example.config.json @@ -1,5 +1,8 @@ { "token": "", + "refresh_token": "", + "client_id": "", + "client_secret": "", "start_date": "2018-01-01T00:00:00Z", "forms": "ZFuC6U,bFPlvG,WFBGBZ,WF0XE6,xFWoCE,OFHRwO,QFh3FI", "page_size": 100, From 982eeeed57e61b7066b8b2e6ff6505b25e6310ca Mon Sep 17 00:00:00 2001 From: RushiT0122 Date: Mon, 27 Nov 2023 12:46:46 +0000 Subject: [PATCH 09/10] add the offline scope --- tap_typeform/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_typeform/client.py b/tap_typeform/client.py index 8181382..81f8052 100644 --- a/tap_typeform/client.py +++ b/tap_typeform/client.py @@ -167,7 +167,7 @@ def refresh(self): 'client_secret': self.client_secret, 'refresh_token': self.refresh_token, 'grant_type': 'refresh_token', - 'scope': 'forms:read accounts:read images:read responses:read themes:read workspaces:read'}) + 'scope': 'forms:read accounts:read images:read responses:read themes:read workspaces:read offline'}) if response.status_code != 200: raise_for_error(response) From de20f5ab81e73b6f516306efed87fc2b1176439e Mon Sep 17 00:00:00 2001 From: Rushikesh Todkar <98420315+RushiT0122@users.noreply.github.com> Date: Thu, 30 Nov 2023 14:41:25 +0530 Subject: [PATCH 10/10] Add token chaining in integration tests (#77) * add token chaining in integration tests --------- Co-authored-by: RushiT0122 --- tests/base.py | 15 ++++++++++++++- tests/test_typeform_all_fields.py | 4 ++-- tests/test_typeform_automatic_fields.py | 4 ++-- tests/test_typeform_bookmarks.py | 6 +++--- tests/test_typeform_discovery.py | 4 ++-- tests/test_typeform_pagination.py | 5 +++-- tests/test_typeform_start_date.py | 8 ++++---- 7 files changed, 30 insertions(+), 16 deletions(-) diff --git a/tests/base.py b/tests/base.py index d8674a2..9a8b51a 100644 --- a/tests/base.py +++ b/tests/base.py @@ -47,6 +47,7 @@ def get_type(): def get_properties(self, original: bool = True): """Configuration properties required for the tap.""" return_value = { + 'client_id': os.getenv('TAP_TYPEFORM_CLIENT_ID'), 'start_date' : '2021-05-10T00:00:00Z', 'forms': os.getenv('TAP_TYPEFORM_FORMS'), 'incremental_range': 'daily', @@ -64,7 +65,19 @@ def get_forms(self): @staticmethod def get_credentials(): """Authentication information for the test account""" - return {'token': os.getenv('TAP_TYPEFORM_TOKEN')} + return { + 'refresh_token': os.getenv('TAP_TYPEFORM_REFRESH_TOKEN'), + 'token': os.getenv('TAP_TYPEFORM_TOKEN'), + 'client_secret': os.getenv('TAP_TYPEFORM_CLIENT_SECRET')} + + @staticmethod + def preserve_refresh_token(existing_conns, payload): + """This method is used get the refresh token from an existing refresh token""" + if not existing_conns: + return payload + conn_with_creds = connections.fetch_existing_connection_with_creds(existing_conns[0]['id']) + payload['properties']['refresh_token'] = conn_with_creds['credentials'].get('refresh_token') + return payload def expected_metadata(self): """The expected streams and metadata about the streams""" diff --git a/tests/test_typeform_all_fields.py b/tests/test_typeform_all_fields.py index aa84dc9..4a01b9b 100644 --- a/tests/test_typeform_all_fields.py +++ b/tests/test_typeform_all_fields.py @@ -17,7 +17,7 @@ class TypeformAllFieldsTest(TypeformBaseTest): """Ensure running the tap with all streams and fields selected results in the replication of all fields.""" def name(self): - return "tap_tester_typeform_all_fields_test" + return "tap_tester_typeform_using_shared_token_chaining" def test_run(self): """ @@ -29,7 +29,7 @@ def test_run(self): # Streams to verify all fields tests streams_to_test = self.expected_streams() - conn_id = connections.ensure_connection(self) + conn_id = connections.ensure_connection(self, payload_hook=self.preserve_refresh_token) expected_automatic_fields = self.expected_automatic_fields() diff --git a/tests/test_typeform_automatic_fields.py b/tests/test_typeform_automatic_fields.py index e0d04e1..550de45 100644 --- a/tests/test_typeform_automatic_fields.py +++ b/tests/test_typeform_automatic_fields.py @@ -13,7 +13,7 @@ class TypeformAutomaticFields(TypeformBaseTest): @staticmethod def name(): - return "tap_tester_typeform_automatic_fields" + return "tap_tester_typeform_using_shared_token_chaining" def test_run(self): """ @@ -32,7 +32,7 @@ def test_run(self): expected_streams = self.expected_streams() # Instantiate connection - conn_id = connections.ensure_connection(self) + conn_id = connections.ensure_connection(self, payload_hook=self.preserve_refresh_token) # Run check mode found_catalogs = self.run_and_verify_check_mode(conn_id) diff --git a/tests/test_typeform_bookmarks.py b/tests/test_typeform_bookmarks.py index bace469..43f5857 100644 --- a/tests/test_typeform_bookmarks.py +++ b/tests/test_typeform_bookmarks.py @@ -12,7 +12,7 @@ class TypeformBookmarks(TypeformBaseTest): @staticmethod def name(): - return "tap_tester_typeform_bookmarks" + return "tap_tester_typeform_using_shared_token_chaining" @staticmethod def convert_state_to_utc(date_str): @@ -39,7 +39,7 @@ def test_run(self): self.start_date_2 = self.timedelta_formatted(self.start_date_1, days=3) self.start_date = self.start_date_1 - conn_id = connections.ensure_connection(self, original_properties=False) + conn_id = connections.ensure_connection(self, original_properties=False, payload_hook=self.preserve_refresh_token) # Run in check mode found_catalogs = self.run_and_verify_check_mode(conn_id) @@ -72,7 +72,7 @@ def test_run(self): } for stream, new_state in simulated_states.items(): new_states['bookmarks'][stream] = new_state - conn_id_2 = connections.ensure_connection(self, original_properties=False) + conn_id_2 = connections.ensure_connection(self, original_properties=False, payload_hook=self.preserve_refresh_token) menagerie.set_state(conn_id_2, new_states) for stream in simulated_states.keys(): diff --git a/tests/test_typeform_discovery.py b/tests/test_typeform_discovery.py index 39be4a9..f76480a 100644 --- a/tests/test_typeform_discovery.py +++ b/tests/test_typeform_discovery.py @@ -11,7 +11,7 @@ class DiscoveryTest(TypeformBaseTest): @staticmethod def name(): - return "tap_tester_typeform_discovery_test" + return "tap_tester_typeform_using_shared_token_chaining" def test_run(self): """ @@ -32,7 +32,7 @@ def test_run(self): """ streams_to_test = self.expected_streams() - conn_id = connections.ensure_connection(self) + conn_id = connections.ensure_connection(self, payload_hook=self.preserve_refresh_token) # Verify that there are catalogs found found_catalogs = self.run_and_verify_check_mode(conn_id) diff --git a/tests/test_typeform_pagination.py b/tests/test_typeform_pagination.py index 46f22cd..892c583 100644 --- a/tests/test_typeform_pagination.py +++ b/tests/test_typeform_pagination.py @@ -11,11 +11,12 @@ class TypeformPaginationTest(TypeformBaseTest): """ def name(self): - return "tap_tester_typeform_pagination_test" + return "tap_tester_typeform_using_shared_token_chaining" def get_properties(self, original: bool = True): """Configuration properties required for the tap.""" return_value = { + 'client_id': os.getenv('TAP_TYPEFORM_CLIENT_ID'), 'start_date' : '2021-05-10T00:00:00Z', 'forms': os.getenv('TAP_TYPEFORM_FORMS'), 'incremental_range': 'daily', @@ -44,7 +45,7 @@ def run_test(self, expected_streams, page_size): self.PAGE_SIZE = page_size - conn_id = connections.ensure_connection(self) + conn_id = connections.ensure_connection(self, payload_hook=self.preserve_refresh_token) # Verify that there are catalogs found found_catalogs = self.run_and_verify_check_mode(conn_id) diff --git a/tests/test_typeform_start_date.py b/tests/test_typeform_start_date.py index 83dcc45..0d05b80 100644 --- a/tests/test_typeform_start_date.py +++ b/tests/test_typeform_start_date.py @@ -12,7 +12,7 @@ class TypeformStartDateTest(TypeformBaseTest): @staticmethod def name(): - return "tap_tester_typeform_start_date_test" + return "tap_tester_typeform_using_shared_token_chaining" start_date_1 = "" start_date_2 = "" @@ -39,7 +39,7 @@ def test_run(self): ########################################################################## # Instantiate connection - conn_id_1 = connections.ensure_connection(self) + conn_id_1 = connections.ensure_connection(self, payload_hook=self.preserve_refresh_token) # Run check mode found_catalogs_1 = self.run_and_verify_check_mode(conn_id_1) @@ -65,7 +65,7 @@ def test_run(self): ########################################################################## # Create a new connection with the new start_date - conn_id_2 = connections.ensure_connection(self, original_properties=False) + conn_id_2 = connections.ensure_connection(self, original_properties=False, payload_hook=self.preserve_refresh_token) # Run check mode found_catalogs_2 = self.run_and_verify_check_mode(conn_id_2) @@ -150,4 +150,4 @@ def test_run(self): self.assertEqual(record_count_sync_2, record_count_sync_1) # Verify by primary key values that the same records are replicated in the 1st and 2nd syncs - self.assertSetEqual(primary_keys_sync_1, primary_keys_sync_2) \ No newline at end of file + self.assertSetEqual(primary_keys_sync_1, primary_keys_sync_2)