Log hours

meltano.yml

@@ -25,10 +25,14 @@ plugins:
       value: '2010-01-01T00:00:00Z'
     config:
       start_date: '2010-01-01T00:00:00Z'
-      #api_url: https://sprintsapi.zoho.com.au/zsapi
-      api_url: https://sprintsapi.zoho.com/zsapi
-      oauth_url: https://accounts.zoho.com/oauth/v2/token
-      #oauth_url: https://accounts.zoho.com.au/oauth/v2/token
+      api_url: https://sprintsapi.zoho.com.au/zsapi
+      oauth_url: https://accounts.zoho.com.au/oauth/v2/token
+      #api_url: https://sprintsapi.zoho.com/zsapi
+      #oauth_url: https://accounts.zoho.com/oauth/v2/token
+    select:
+    - '!log_hour.*'
+    - '!project_user.*'
+    - '*.*'
   loaders:
   - name: target-jsonl
     variant: andyh1203

tap_zohosprints/client.py

@@ -5,7 +5,7 @@
 import requests
 import time
 from pathlib import Path
-from typing import Any, Dict, Optional, Union, List, Iterable, cast
+from typing import Any, Dict, Optional, Union, List, Iterable, cast, Generator
 
 from memoization import cached
 
@@ -142,21 +142,6 @@ def validate_response(self, response):
         if data.get("code") == 7602.1:
             raise FatalAPIError("Error, locked out of the API")
 
-        # Still catch error status codes
-        super().validate_response(response)
-
-    def validate_response(self, response: requests.Response) -> None:
-        """Validate HTTP response.
-        Args:
-            response: A `requests.Response`_ object.
-
-        Raises:
-            FatalAPIError: If the request is not retriable.
-            RetriableAPIError: If the request is retriable.
-
-        .. _requests.Response:
-            https://docs.python-requests.org/en/latest/api/#requests.Response
-        """
         msg = (
             f"{response.status_code} Client Error: "
             f"{response.reason} for path: {self.path}"
@@ -217,7 +202,7 @@ def property_unfurler(
     ids_key: str,
     jobj_key: str,
     primary_key_name: str,
-) -> Iterable[dict]:
+) -> Generator[dict, None, None]:
     """
     Zohosprints embeds data inside of a JObj key.
 
@@ -249,17 +234,20 @@ def property_unfurler(
     json = response.json()
     props: Dict = json.get(prop_key)
     ids: List = json.get(ids_key)
-    for id in ids:
-        record = {}
-        prop_values: List = json[jobj_key][id]
-        for property_name, property_index in props.items():
-            record[property_name] = prop_values[property_index]
+    if json.get("hasData") == False:
         return_object: Dict = copy.deepcopy(json)
-        return_object[primary_key_name] = id
-        return_object.pop(prop_key)
-        return_object.pop(ids_key)
-        return_object.pop(jobj_key)
-        return_object["record"] = record
-        yield return_object
-
+        yield return_object  # Data is empty, stop.
+    else:
+        for id in ids:
+            record = {}
+            prop_values: List = json[jobj_key][id]
+            for property_name, property_index in props.items():
+                record[property_name] = prop_values[property_index]
+            return_object: Dict = copy.deepcopy(json)
+            return_object[primary_key_name] = id
+            return_object.pop(prop_key)
+            return_object.pop(ids_key)
+            return_object.pop(jobj_key)
+            return_object["record"] = record
+            yield return_object
     return {}

tap_zohosprints/schemas/log_hour.json

@@ -0,0 +1,44 @@
+{
+    "type": "object",
+    "properties": {
+        "logIds": {
+            "type": "array",
+            "items": {
+                "type": "string"
+            }
+        },
+        "hasData": {
+            "type": "boolean"
+        },
+        "userDisplayName": {
+            "type": "object"
+        },
+        "hasNext": {
+            "type": "boolean"
+        },
+        "nextIndex": {
+            "type": "integer"
+        },
+        "zsuserIdvsZUID": {
+            "type": "object"
+        },
+        "status": {
+            "type": "string"
+        },
+        "record": {
+            "type": "object"
+        },
+        "tLogId": {
+            "type": "string"
+        },
+        "team_id": {
+            "type": "string"
+        },
+        "project_id": {
+            "type": "string"
+        },
+        "sprint_id": {
+            "type": "string"
+        }
+    }
+}

tap_zohosprints/streams.py

@@ -420,3 +420,25 @@ def parse_response(self, response: requests.Response) -> Iterable[dict]:
             jobj_key="userJObj",
             primary_key_name="userId",
         )
+
+
+class LogHours(ZohoSprintsPropsStream):
+    """Log Hours Stream"""
+
+    name = "log_hour"
+    path = "/team/{team_id}/projects/{project_id}/sprints/{sprint_id}/timesheet/?action=logitems&listviewtype=0"
+    parent_stream_type = SprintsStream
+    primary_keys = ["tLogId"]
+    replication_key = None
+    schema_filepath = SCHEMAS_DIR / "log_hour.json"
+
+    def parse_response(self, response: requests.Response) -> Iterable[dict]:
+        """Parse the response and return an iterator of result rows."""
+        # Create a record object
+        yield from property_unfurler(
+            response=response,
+            prop_key="log_prop",
+            ids_key="logIds",
+            jobj_key="logJObj",
+            primary_key_name="tLogId",
+        )

tap_zohosprints/tap.py

@@ -20,6 +20,7 @@
     TagsStream,
     SprintUsers,
     ProjectUsers,
+    LogHours,
 )
 
 # TODO: Compile a list of custom stream types here
@@ -38,6 +39,7 @@
     TagsStream,
     SprintUsers,
     ProjectUsers,
+    LogHours,
 ]
 
 

tap_zohosprints/tests/tag_property_unfurl_hasnodata.json

@@ -0,0 +1,4 @@
+{
+    "hasData": false,
+    "status": "success"
+}

tap_zohosprints/tests/test_core.py

@@ -52,7 +52,6 @@ def test_property_unfurler(mocked_responses):
         content_type="application/json",
     )
     resp = requests.get("https://autoidm.com")
-    assert resp.status_code == 200
 
     unfurled = property_unfurler(
         response=resp,
@@ -79,3 +78,35 @@ def test_property_unfurler(mocked_responses):
         },
         "tagId": "114398000000007021",
     }
+
+
+def test_property_unfurler_nodata(mocked_responses):
+    """ Sometimes the data comes back empty, we need to not attempt to unfurl the data """
+    tag_json = ""
+    # Should probably just define the json inline as it's way easier to read
+    with open(
+        Path(__file__).parent / Path("tag_property_unfurl_hasnodata.json")
+    ) as tag:
+        tag_json = tag.read()
+
+    mocked_responses.add(
+        responses.GET,
+        "https://autoidm.com",
+        body=tag_json,
+        status=200,
+        content_type="application/json",
+    )
+    resp = requests.get("https://autoidm.com")
+
+    unfurled = property_unfurler(
+        response=resp,
+        prop_key="abcdef",
+        ids_key="hijklmn",
+        jobj_key="opqrst",
+        primary_key_name="tuVWxYZ",
+    )
+    output = None
+    for data in unfurled:
+        print(data)
+        output = data
+    assert output == {"hasData": False, "status": "success"}