Enable dev mode

example.config.json

@@ -1,5 +1,8 @@
 {
   "token": "<myapitoken>",
+  "refresh_token": "<my_refresh_token>",
+  "client_id": "<my_client_id>",
+  "client_secret": "<my_client_secret>",
   "start_date": "2018-01-01T00:00:00Z",
   "forms": "ZFuC6U,bFPlvG,WFBGBZ,WF0XE6,xFWoCE,OFHRwO,QFh3FI",
   "page_size": 100,

setup.py

@@ -10,7 +10,7 @@
     url="http://singer.io",
     classifiers=["Programming Language :: Python :: 3 :: Only"],
     install_requires=[
-        "singer-python==5.10.0",
+        "singer-python==5.13.0",
         "pendulum",
         "ratelimit",
         "backoff",

tap_typeform/__init__.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 import singer
-from singer import utils
+from singer import utils as _utils
 from tap_typeform.discover import discover as _discover
 from tap_typeform.sync import sync as _sync
 from tap_typeform.client import Client
@@ -34,11 +34,15 @@ def validate_form_ids(client, config):
     return config_forms
 
 
-@utils.handle_top_exception(LOGGER)
+@_utils.handle_top_exception(LOGGER)
 def main():
-    args = utils.parse_args(REQUIRED_CONFIG_KEYS)
+    args = _utils.parse_args(REQUIRED_CONFIG_KEYS)
     config = args.config
-    client = Client(config)
+
+    if args.dev:
+        LOGGER.warning("Executing Tap in Dev mode")
+
+    client = Client(config, args.config_path, args.dev)
     valid_forms = validate_form_ids(client, config)
     if args.discover:
         catalog = _discover()

tap_typeform/client.py

@@ -2,7 +2,11 @@
 import backoff
 import singer
 
+from datetime import timedelta
+from singer.utils import now
 from requests.exceptions import ChunkedEncodingError, Timeout, ConnectionError
+from tap_typeform.utils import write_config
+
 
 LOGGER = singer.get_logger()
 
@@ -113,22 +117,69 @@ class Client(object):
     The client class is used for making REST calls to the Github API.
     """
     BASE_URL = 'https://api.typeform.com'
+    OAUTH_URL = 'https://api.typeform.com/oauth/token'
 
-    def __init__(self, config):
-        self.token = 'Bearer ' + config.get('token')
+    def __init__(self, config, config_path, dev_mode):
         self.metric = config.get('metric')
         self.session = requests.Session()
         self.page_size = MAX_RESPONSES_PAGE_SIZE
         self.form_page_size = FORMS_PAGE_SIZE
+        self.config_path = config_path
         self.get_page_size(config)
 
+        self.client_id = config.get('client_id')
+        self.client_secret = config.get('client_secret')
+        self.refresh_token = config.get('refresh_token')
+        self.access_token = config.get('token')
+        self.dev_mode = dev_mode
+        self.refresh()
+
         # Set and pass request timeout to config param `request_timeout` value.
         config_request_timeout = config.get('request_timeout')
         if config_request_timeout and float(config_request_timeout):
             self.request_timeout = float(config_request_timeout)
         else:
             self.request_timeout = REQUEST_TIMEOUT # If value is 0,"0","" or not passed then it set default to 300 seconds.
 
+    @backoff.on_exception(backoff.expo,(Timeout, ConnectionError),  # Backoff for Timeout and ConnectionError.
+                            max_tries=5, factor=2, jitter=None)
+    @backoff.on_exception(backoff.expo, (TypeformInternalError, TypeformNotAvailableError, TypeformTooManyError, ChunkedEncodingError),
+                            max_tries=3, factor=2)
+    def refresh(self):
+        """
+        Refreshes access token and refresh token
+        """
+        # Existing connections won't have refresh token so use the existing access token
+        if not self.refresh_token:
+            return
+
+        # In dev mode, don't refresh access token
+        if self.dev_mode:
+            if not self.access_token:
+                raise Exception('Access token is missing')
+
+            return
+
+        response = self.session.post(url=self.OAUTH_URL,
+                                     headers={
+                                         'Content-Type': 'application/x-www-form-urlencoded'},
+                                     data={'client_id': self.client_id,
+                                           'client_secret': self.client_secret,
+                                           'refresh_token': self.refresh_token,
+                                           'grant_type': 'refresh_token',
+                                           'scope': 'forms:read accounts:read images:read responses:read themes:read workspaces:read offline'})
+
+        if response.status_code != 200:
+            raise_for_error(response)
+
+        data = response.json()
+        self.refresh_token = data['refresh_token']
+        self.access_token = data['access_token']
+
+        write_config(self.config_path,
+                     {'refresh_token': self.refresh_token,
+                      'token': self.access_token})
+
     def get_page_size(self, config):
         """
         This function will get page size from config,
@@ -138,7 +189,7 @@ def get_page_size(self, config):
         if page_size is None:
             return
         if ((type(page_size) == int or type(page_size) == float) and (page_size > 0)) or \
-            (type(page_size) == str and page_size.replace('.', '', 1).isdigit() and (float(page_size) > 0) ):
+                (type(page_size) == str and page_size.replace('.', '', 1).isdigit() and (float(page_size) > 0)):
             self.page_size = int(float(page_size))
             self.form_page_size = min(self.form_page_size, self.page_size)
         else:
@@ -150,19 +201,19 @@ def build_url(self, endpoint):
         """
         return f"{self.BASE_URL}/{endpoint}"
 
-    @backoff.on_exception(backoff.expo,(Timeout, ConnectionError), # Backoff for Timeout and ConnectionError.
-                            max_tries=5, factor=2, jitter=None)
+    @backoff.on_exception(backoff.expo, (Timeout, ConnectionError),  # Backoff for Timeout and ConnectionError.
+                          max_tries=5, factor=2, jitter=None)
     @backoff.on_exception(backoff.expo, (TypeformInternalError, TypeformNotAvailableError, TypeformTooManyError, ChunkedEncodingError),
-                            max_tries=3, factor=2)
+                          max_tries=3, factor=2)
     def request(self, url, params={}, **kwargs):
         """
         Call rest API and return the response in case of status code 200.
         """
 
         if 'headers' not in kwargs:
             kwargs['headers'] = {}
-        if self.token:
-            kwargs['headers']['Authorization'] = self.token
+        if self.access_token:
+            kwargs['headers']['Authorization'] = 'Bearer ' + self.access_token
 
         LOGGER.info("URL: %s and Params: %s", url, params)
         response = self.session.get(url, params=params, headers=kwargs['headers'], timeout=self.request_timeout)

tap_typeform/utils.py

@@ -0,0 +1,24 @@
+import json
+
+
+def read_config(config_path):
+    """
+    Performs read on the provided filepath,
+    returns empty dict if invalid path provided
+    """
+    try:
+        with open(config_path, "r") as tap_config:
+            return json.load(tap_config)
+    except FileNotFoundError as err:
+        raise Exception("Failed to load config in dev mode") from err
+
+
+def write_config(config_path, data):
+    """
+    Updates the provided filepath with json format of the `data` object
+    """
+    config = read_config(config_path)
+    config.update(data)
+    with open(config_path, "w") as tap_config:
+        json.dump(config, tap_config, indent=2)
+    return config

tests/base.py

@@ -47,6 +47,7 @@ def get_type():
     def get_properties(self, original: bool = True):
         """Configuration properties required for the tap."""
         return_value = {
+            'client_id': os.getenv('TAP_TYPEFORM_CLIENT_ID'),
             'start_date' : '2021-05-10T00:00:00Z',
             'forms': os.getenv('TAP_TYPEFORM_FORMS'),
             'incremental_range': 'daily',
@@ -64,7 +65,19 @@ def get_forms(self):
     @staticmethod
     def get_credentials():
         """Authentication information for the test account"""
-        return {'token': os.getenv('TAP_TYPEFORM_TOKEN')}
+        return {
+            'refresh_token': os.getenv('TAP_TYPEFORM_REFRESH_TOKEN'),
+            'token': os.getenv('TAP_TYPEFORM_TOKEN'),
+            'client_secret': os.getenv('TAP_TYPEFORM_CLIENT_SECRET')}
+
+    @staticmethod
+    def preserve_refresh_token(existing_conns, payload):
+        """This method is used get the refresh token from an existing refresh token"""
+        if not existing_conns:
+            return payload
+        conn_with_creds = connections.fetch_existing_connection_with_creds(existing_conns[0]['id'])
+        payload['properties']['refresh_token'] = conn_with_creds['credentials'].get('refresh_token')
+        return payload
 
     def expected_metadata(self):
         """The expected streams and metadata about the streams"""

tests/test_typeform_all_fields.py

@@ -17,7 +17,7 @@ class TypeformAllFieldsTest(TypeformBaseTest):
     """Ensure running the tap with all streams and fields selected results in the replication of all fields."""
 
     def name(self):
-        return "tap_tester_typeform_all_fields_test"
+        return "tap_tester_typeform_using_shared_token_chaining"
 
     def test_run(self):
         """
@@ -29,7 +29,7 @@ def test_run(self):
         # Streams to verify all fields tests
         streams_to_test = self.expected_streams()
 
-        conn_id = connections.ensure_connection(self)
+        conn_id = connections.ensure_connection(self, payload_hook=self.preserve_refresh_token)
 
         expected_automatic_fields = self.expected_automatic_fields()
 

tests/test_typeform_automatic_fields.py

@@ -13,7 +13,7 @@ class TypeformAutomaticFields(TypeformBaseTest):
 
     @staticmethod
     def name():
-        return "tap_tester_typeform_automatic_fields"
+        return "tap_tester_typeform_using_shared_token_chaining"
 
     def test_run(self):
         """
@@ -32,7 +32,7 @@ def test_run(self):
         expected_streams = self.expected_streams()
 
         # Instantiate connection
-        conn_id = connections.ensure_connection(self)
+        conn_id = connections.ensure_connection(self, payload_hook=self.preserve_refresh_token)
 
         # Run check mode
         found_catalogs = self.run_and_verify_check_mode(conn_id)

tests/test_typeform_bookmarks.py

@@ -12,7 +12,7 @@ class TypeformBookmarks(TypeformBaseTest):
 
     @staticmethod
     def name():
-        return "tap_tester_typeform_bookmarks"
+        return "tap_tester_typeform_using_shared_token_chaining"
 
     @staticmethod
     def convert_state_to_utc(date_str):
@@ -39,7 +39,7 @@ def test_run(self):
         self.start_date_2 = self.timedelta_formatted(self.start_date_1, days=3)
 
         self.start_date = self.start_date_1
-        conn_id = connections.ensure_connection(self, original_properties=False)
+        conn_id = connections.ensure_connection(self, original_properties=False, payload_hook=self.preserve_refresh_token)
 
         # Run in check mode
         found_catalogs = self.run_and_verify_check_mode(conn_id)
@@ -72,7 +72,7 @@ def test_run(self):
         }
         for stream, new_state in simulated_states.items():
             new_states['bookmarks'][stream] = new_state
-        conn_id_2 = connections.ensure_connection(self, original_properties=False)
+        conn_id_2 = connections.ensure_connection(self, original_properties=False, payload_hook=self.preserve_refresh_token)
         menagerie.set_state(conn_id_2, new_states)
 
         for stream in simulated_states.keys():

tests/test_typeform_discovery.py

@@ -11,7 +11,7 @@ class DiscoveryTest(TypeformBaseTest):
 
     @staticmethod
     def name():
-        return "tap_tester_typeform_discovery_test"
+        return "tap_tester_typeform_using_shared_token_chaining"
 
     def test_run(self):
         """
@@ -32,7 +32,7 @@ def test_run(self):
         """
         streams_to_test = self.expected_streams()
 
-        conn_id = connections.ensure_connection(self)
+        conn_id = connections.ensure_connection(self, payload_hook=self.preserve_refresh_token)
 
         # Verify that there are catalogs found
         found_catalogs = self.run_and_verify_check_mode(conn_id)

tests/test_typeform_pagination.py

@@ -11,11 +11,12 @@ class TypeformPaginationTest(TypeformBaseTest):
     """
 
     def name(self):
-        return "tap_tester_typeform_pagination_test"
+        return "tap_tester_typeform_using_shared_token_chaining"
 
     def get_properties(self, original: bool = True):
         """Configuration properties required for the tap."""
         return_value = {
+            'client_id': os.getenv('TAP_TYPEFORM_CLIENT_ID'),
             'start_date' : '2021-05-10T00:00:00Z',
             'forms': os.getenv('TAP_TYPEFORM_FORMS'),
             'incremental_range': 'daily',
@@ -44,7 +45,7 @@ def run_test(self, expected_streams, page_size):
 
         self.PAGE_SIZE = page_size
 
-        conn_id = connections.ensure_connection(self)
+        conn_id = connections.ensure_connection(self, payload_hook=self.preserve_refresh_token)
 
         # Verify that there are catalogs found
         found_catalogs = self.run_and_verify_check_mode(conn_id)

tests/test_typeform_start_date.py

@@ -12,7 +12,7 @@ class TypeformStartDateTest(TypeformBaseTest):
 
     @staticmethod
     def name():
-        return "tap_tester_typeform_start_date_test"
+        return "tap_tester_typeform_using_shared_token_chaining"
 
     start_date_1 = ""
     start_date_2 = ""
@@ -39,7 +39,7 @@ def test_run(self):
         ##########################################################################
 
         # Instantiate connection
-        conn_id_1 = connections.ensure_connection(self)
+        conn_id_1 = connections.ensure_connection(self, payload_hook=self.preserve_refresh_token)
 
         # Run check mode
         found_catalogs_1 = self.run_and_verify_check_mode(conn_id_1)
@@ -65,7 +65,7 @@ def test_run(self):
         ##########################################################################
 
         # Create a new connection with the new start_date
-        conn_id_2 = connections.ensure_connection(self, original_properties=False)
+        conn_id_2 = connections.ensure_connection(self, original_properties=False, payload_hook=self.preserve_refresh_token)
 
         # Run check mode
         found_catalogs_2 = self.run_and_verify_check_mode(conn_id_2)
@@ -150,4 +150,4 @@ def test_run(self):
                     self.assertEqual(record_count_sync_2, record_count_sync_1)
 
                     # Verify by primary key values that the same records are replicated in the 1st and 2nd syncs
-                    self.assertSetEqual(primary_keys_sync_1, primary_keys_sync_2)
+                    self.assertSetEqual(primary_keys_sync_1, primary_keys_sync_2)