Enable dev mode

example.config.json

@@ -1,5 +1,8 @@
 {
   "token": "<myapitoken>",
+  "refresh_token": "<my_refresh_token>",
+  "client_id": "<my_client_id>",
+  "client_secret": "<my_client_secret>",
   "start_date": "2018-01-01T00:00:00Z",
   "forms": "ZFuC6U,bFPlvG,WFBGBZ,WF0XE6,xFWoCE,OFHRwO,QFh3FI",
   "page_size": 100,

setup.py

@@ -10,7 +10,7 @@
     url="http://singer.io",
     classifiers=["Programming Language :: Python :: 3 :: Only"],
     install_requires=[
-        "singer-python==5.10.0",
+        "singer-python==5.13.0",
         "pendulum",
         "ratelimit",
         "backoff",

tap_typeform/__init__.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 import singer
-from singer import utils
+from singer import utils as _utils
 from tap_typeform.discover import discover as _discover
 from tap_typeform.sync import sync as _sync
 from tap_typeform.client import Client
@@ -34,11 +34,15 @@ def validate_form_ids(client, config):
     return config_forms
 
 
-@utils.handle_top_exception(LOGGER)
+@_utils.handle_top_exception(LOGGER)
 def main():
-    args = utils.parse_args(REQUIRED_CONFIG_KEYS)
+    args = _utils.parse_args(REQUIRED_CONFIG_KEYS)
     config = args.config
-    client = Client(config)
+
+    if args.dev:
+        LOGGER.warning("Executing Tap in Dev mode")
+
+    client = Client(config, args.config_path, args.dev)
     valid_forms = validate_form_ids(client, config)
     if args.discover:
         catalog = _discover()

tap_typeform/client.py

@@ -2,7 +2,11 @@
 import backoff
 import singer
 
+from datetime import timedelta
+from singer.utils import now
 from requests.exceptions import ChunkedEncodingError, Timeout, ConnectionError
+from tap_typeform.utils import write_config
+
 
 LOGGER = singer.get_logger()
 
@@ -113,22 +117,69 @@ class Client(object):
     The client class is used for making REST calls to the Github API.
     """
     BASE_URL = 'https://api.typeform.com'
+    OAUTH_URL = 'https://api.typeform.com/oauth/token'
 
-    def __init__(self, config):
-        self.token = 'Bearer ' + config.get('token')
+    def __init__(self, config, config_path, dev_mode):
         self.metric = config.get('metric')
         self.session = requests.Session()
         self.page_size = MAX_RESPONSES_PAGE_SIZE
         self.form_page_size = FORMS_PAGE_SIZE
+        self.config_path = config_path
         self.get_page_size(config)
 
+        self.client_id = config.get('client_id')
+        self.client_secret = config.get('client_secret')
+        self.refresh_token = config.get('refresh_token')
+        self.access_token = config.get('token')
+        self.dev_mode = dev_mode
+        self.refresh()
+
         # Set and pass request timeout to config param `request_timeout` value.
         config_request_timeout = config.get('request_timeout')
         if config_request_timeout and float(config_request_timeout):
             self.request_timeout = float(config_request_timeout)
         else:
             self.request_timeout = REQUEST_TIMEOUT # If value is 0,"0","" or not passed then it set default to 300 seconds.
 
+    @backoff.on_exception(backoff.expo,(Timeout, ConnectionError),  # Backoff for Timeout and ConnectionError.
+                            max_tries=5, factor=2, jitter=None)
+    @backoff.on_exception(backoff.expo, (TypeformInternalError, TypeformNotAvailableError, TypeformTooManyError, ChunkedEncodingError),
+                            max_tries=3, factor=2)
+    def refresh(self):
+        """
+        Refreshes access token and refresh token
+        """
+        # Existing connections won't have refresh token so use the existing access token
+        if not self.refresh_token:
+            return
+
+        # In dev mode, don't refresh access token
+        if self.dev_mode:
+            if not self.access_token:
+                raise Exception('Access token is missing')
+
+            return
+
+        response = self.session.post(url=self.OAUTH_URL,
+                                     headers={
+                                         'Content-Type': 'application/x-www-form-urlencoded'},
+                                     data={'client_id': self.client_id,
+                                           'client_secret': self.client_secret,
+                                           'refresh_token': self.refresh_token,
+                                           'grant_type': 'refresh_token',
+                                           'scope': 'forms:read accounts:read images:read responses:read themes:read workspaces:read offline'})
+
+        if response.status_code != 200:
+            raise_for_error(response)
+
+        data = response.json()
+        self.refresh_token = data['refresh_token']
+        self.access_token = data['access_token']
+
+        write_config(self.config_path,
+                     {'refresh_token': self.refresh_token,
+                      'token': self.access_token})
+
     def get_page_size(self, config):
         """
         This function will get page size from config,
@@ -138,7 +189,7 @@ def get_page_size(self, config):
         if page_size is None:
             return
         if ((type(page_size) == int or type(page_size) == float) and (page_size > 0)) or \
-            (type(page_size) == str and page_size.replace('.', '', 1).isdigit() and (float(page_size) > 0) ):
+                (type(page_size) == str and page_size.replace('.', '', 1).isdigit() and (float(page_size) > 0)):
             self.page_size = int(float(page_size))
             self.form_page_size = min(self.form_page_size, self.page_size)
         else:
@@ -150,19 +201,19 @@ def build_url(self, endpoint):
         """
         return f"{self.BASE_URL}/{endpoint}"
 
-    @backoff.on_exception(backoff.expo,(Timeout, ConnectionError), # Backoff for Timeout and ConnectionError.
-                            max_tries=5, factor=2, jitter=None)
+    @backoff.on_exception(backoff.expo, (Timeout, ConnectionError),  # Backoff for Timeout and ConnectionError.
+                          max_tries=5, factor=2, jitter=None)
     @backoff.on_exception(backoff.expo, (TypeformInternalError, TypeformNotAvailableError, TypeformTooManyError, ChunkedEncodingError),
-                            max_tries=3, factor=2)
+                          max_tries=3, factor=2)
     def request(self, url, params={}, **kwargs):
         """
         Call rest API and return the response in case of status code 200.
         """
 
         if 'headers' not in kwargs:
             kwargs['headers'] = {}
-        if self.token:
-            kwargs['headers']['Authorization'] = self.token
+        if self.access_token:
+            kwargs['headers']['Authorization'] = 'Bearer ' + self.access_token
 
         LOGGER.info("URL: %s and Params: %s", url, params)
         response = self.session.get(url, params=params, headers=kwargs['headers'], timeout=self.request_timeout)

tap_typeform/utils.py

@@ -0,0 +1,24 @@
+import json
+
+
+def read_config(config_path):
+    """
+    Performs read on the provided filepath,
+    returns empty dict if invalid path provided
+    """
+    try:
+        with open(config_path, "r") as tap_config:
+            return json.load(tap_config)
+    except FileNotFoundError as err:
+        raise Exception("Failed to load config in dev mode") from err
+
+
+def write_config(config_path, data):
+    """
+    Updates the provided filepath with json format of the `data` object
+    """
+    config = read_config(config_path)
+    config.update(data)
+    with open(config_path, "w") as tap_config:
+        json.dump(config, tap_config, indent=2)
+    return config

tests/unittests/test_dev_mode.py

@@ -0,0 +1,98 @@
+import unittest
+import os
+import json
+import requests
+from unittest import mock
+
+from tap_typeform.client import Client
+
+
+http_response = {"refresh_token": "new_refresh_token",
+                 "access_token": "new_access_token"}
+
+test_config_path = "/tmp/test_config.json"
+
+
+
+def write_new_config_file(**kwargs):
+    test_config = {}
+    with open(test_config_path, "w") as config:
+        for key, value in kwargs.items():
+            test_config[key] = value
+        config.write(json.dumps(test_config))
+
+
+class Mockresponse:
+    """ Mock response object class."""
+
+    def __init__(self, status_code, raise_error, text=""):
+        self.status_code = status_code
+        self.raise_error = raise_error
+        self.text = text
+
+    def raise_for_status(self):
+        if not self.raise_error:
+            return self.status_code
+
+        raise requests.HTTPError("Sample message")
+
+    def json(self):
+        """ Response JSON method."""
+        return self.text
+
+
+def get_mock_http_response(status_code):
+    """Return http mock response."""
+    response = requests.Response()
+    response.status_code = status_code
+    return response
+
+
+def get_response(status_code, raise_error=True, text=""):
+    """ Returns required mock response. """
+    return Mockresponse(status_code, raise_error=raise_error, text=text)
+
+
+class TestDevMode(unittest.TestCase):
+    def tearDown(self):
+        if os.path.isfile(test_config_path):
+            os.remove(test_config_path)
+
+    @mock.patch("requests.Session.request")
+    def test_dev_mode_not_enabled(self, mock_post_request):
+        test_config = {"refresh_token": "old_refresh_token",
+                       "token": "old_access_token"}
+        write_new_config_file(**test_config)
+        mock_post_request.side_effect = [get_response(200, raise_error=False, text=http_response)]
+        client = Client(config=test_config, config_path=test_config_path, dev_mode=False)
+        self.assertEqual(client.refresh_token, "new_refresh_token")
+        self.assertEqual(client.access_token, "new_access_token")
+
+    @mock.patch("requests.Session.request")
+    def test_dev_mode_enabled(self, mock_post_request):
+        test_config = {"refresh_token": "old_refresh_token",
+                       "token": "old_access_token"}
+        write_new_config_file(**test_config)
+        mock_post_request.side_effect = [get_response(200, raise_error=False, text=http_response)]
+        client = Client(config=test_config, config_path=test_config_path, dev_mode=True)
+        self.assertEqual(client.refresh_token, "old_refresh_token")
+        self.assertEqual(client.access_token, "old_access_token")
+
+
+    @mock.patch("requests.Session.request")
+    def test_no_refresh_token_not_dev_mode_enabled(self, mock_post_request):
+        test_config = {"token": "old_access_token"}
+        write_new_config_file(**test_config)
+        mock_post_request.side_effect = [get_response(200, raise_error=False, text=http_response)]
+        client = Client(config=test_config, config_path=test_config_path, dev_mode=False)
+        self.assertIsNone(client.refresh_token)
+        self.assertEqual(client.access_token, "old_access_token")
+
+    @mock.patch("requests.Session.request")
+    def test_no_refresh_token_dev_mode_enabled(self, mock_post_request):
+        test_config = {"token": "old_access_token"}
+        write_new_config_file(**test_config)
+        mock_post_request.side_effect = [get_response(200, raise_error=False, text=http_response)]
+        client = Client(config=test_config, config_path=test_config_path, dev_mode=True)
+        self.assertIsNone(client.refresh_token)
+        self.assertEqual(client.access_token, "old_access_token")