Merge pull request #69 from oviohub/fix-ci

Fix CI

.github/workflows/ci_workflow.yml

@@ -23,7 +23,7 @@ jobs:
     - name: Install Poetry
       uses: snok/install-poetry@v1
       with:
-        version: 1.1.8
+        version: 1.1.13
     - name: Install dependencies
       run: |
         poetry install

tap_gitlab/__init__.py

@@ -0,0 +1 @@
+"""Placeholder."""

tap_gitlab/caching.py

@@ -6,6 +6,7 @@
 
 
 def setup_requests_cache(tap_config: dict) -> None:
+    """Install the caching mechanism for requests."""
     cache_path_root = tap_config.get("requests_cache_path", None)
     if not cache_path_root:
         return None

tap_gitlab/client.py

@@ -2,14 +2,12 @@
 
 from __future__ import annotations
 
-import requests
 from pathlib import Path
+from typing import Any, Dict, List, Optional, cast
 
-from typing import Any, Dict, List, cast, Optional
-
-from singer_sdk.streams import RESTStream
+import requests
 from singer_sdk.authenticators import APIKeyAuthenticator
-
+from singer_sdk.streams import RESTStream
 
 API_TOKEN_KEY = "Private-Token"
 API_TOKEN_SETTING_NAME = "private_token"
@@ -34,10 +32,12 @@ def url_base(self) -> str:
 
     @property
     def schema_filename(self) -> str:
+        """Return the filename for the stream's schema."""
         return f"{self.name}.json"
 
     @property
     def schema_filepath(self) -> Path:
+        """Return the filepath for the stream's schema."""
         return SCHEMAS_DIR / self.schema_filename
 
     @property

tap_gitlab/streams.py

@@ -2,7 +2,7 @@
 
 from typing import Any, Dict, Optional
 
-from tap_gitlab.client import GitLabStream, ProjectBasedStream, GroupBasedStream
+from tap_gitlab.client import GitLabStream, GroupBasedStream, ProjectBasedStream
 from tap_gitlab.transforms import object_array_to_id_array, pop_nested_id
 
 # Project-Specific Streams
@@ -19,6 +19,7 @@ class ProjectsStream(ProjectBasedStream):
     extra_url_params = {"statistics": 1}
 
     def post_process(self, row: dict, context: Optional[dict] = None) -> Optional[dict]:
+        """Post process records."""
         result = super().post_process(row, context)
         if result is None:
             return None
@@ -27,6 +28,7 @@ def post_process(self, row: dict, context: Optional[dict] = None) -> Optional[di
         return result
 
     def get_child_context(self, record: dict, context: Optional[dict]) -> dict:
+        """Perform post processing, including queuing up any child stream types."""
         assert context is not None
         return {
             "project_id": record["id"],
@@ -46,6 +48,7 @@ class IssuesStream(ProjectBasedStream):
     extra_url_params = {"scope": "all"}
 
     def post_process(self, row: dict, context: Optional[dict] = None) -> Optional[dict]:
+        """Post process records."""
         result = super().post_process(row, context)
         if result is None:
             return None
@@ -55,6 +58,8 @@ def post_process(self, row: dict, context: Optional[dict] = None) -> Optional[di
 
 
 class ProjectMergeRequestsStream(ProjectBasedStream):
+    """Gitlab Merge Requests stream."""
+
     name = "merge_requests"
     path = "/projects/{project_path}/merge_requests"
     primary_keys = ["id"]
@@ -73,6 +78,7 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict:
         }
 
     def post_process(self, row: dict, context: Optional[dict] = None) -> Optional[dict]:
+        """Post process records."""
         result = super().post_process(row, context)
         if result is None:
             return None
@@ -106,13 +112,16 @@ class CommitsStream(ProjectBasedStream):
 
 
 class BranchesStream(ProjectBasedStream):
+    """Gitlab Branches stream."""
+
     name = "branches"
     path = "/projects/{project_path}/repository/branches"
     primary_keys = ["project_id", "name"]
     # TODO: Research why this fails:
     # parent_stream_type = ProjectsStream
 
     def post_process(self, row: dict, context: Optional[dict] = None) -> Optional[dict]:
+        """Post process records."""
         result = super().post_process(row, context)
         if result is None:
             return None
@@ -126,33 +135,42 @@ def post_process(self, row: dict, context: Optional[dict] = None) -> Optional[di
 
 
 class PipelinesStream(ProjectBasedStream):
+    """Gitlab Pipelines stream."""
+
     name = "pipelines"
     path = "/projects/{project_path}/pipelines"
     primary_keys = ["id"]
     replication_key = "updated_at"
     bookmark_param_name = "updated_after"
 
     def get_child_context(self, record: dict, context: Optional[dict]) -> dict:
+        """Perform post processing, including queuing up any child stream types."""
         context = super().get_child_context(record, context)
         context["pipeline_id"] = record["id"]
         return context
 
 
 class PipelinesExtendedStream(ProjectBasedStream):
+    """Gitlab extended Pipelines stream."""
+
     name = "pipelines_extended"
     path = "/projects/{project_path}/pipelines/{pipeline_id}"
     primary_keys = ["id"]
     parent_stream_type = PipelinesStream
 
 
 class PipelineJobsStream(ProjectBasedStream):
+    """Gitlab Pipeline Jobs stream."""
+
     name = "jobs"
     path = "/projects/{project_path}/pipelines/{pipeline_id}/jobs"
     primary_keys = ["id"]
     parent_stream_type = PipelinesStream  # Stream should wait for parents to complete.
 
 
 class ProjectMilestonesStream(ProjectBasedStream):
+    """Gitlab Project Milestones stream."""
+
     name = "project_milestones"
     path = "/projects/{project_path}/milestones"
     primary_keys = ["id"]
@@ -169,30 +187,40 @@ class MergeRequestCommitsStream(ProjectBasedStream):
 
 
 class ProjectUsersStream(ProjectBasedStream):
+    """Gitlab Project Users stream."""
+
     name = "users"
     path = "/projects/{project_path}/users"
     primary_keys = ["id"]
 
 
 class ProjectMembersStream(ProjectBasedStream):
+    """Gitlab Project Members stream."""
+
     name = "project_members"
     path = "/projects/{project_path}/members"
     primary_keys = ["project_id", "id"]
 
 
 class ProjectLabelsStream(ProjectBasedStream):
+    """Gitlab Project Labels stream."""
+
     name = "project_labels"
     path = "/projects/{project_path}/labels"
     primary_keys = ["project_id", "id"]
 
 
 class ProjectVulnerabilitiesStream(ProjectBasedStream):
+    """Project Vulnerabilities stream."""
+
     name = "vulnerabilities"
     path = "/projects/{project_path}/vulnerabilities"
     primary_keys = ["id"]
 
 
 class ProjectVariablesStream(ProjectBasedStream):
+    """Project Variables stream."""
+
     name = "project_variables"
     path = "/projects/{project_path}/variables"
     primary_keys = ["group_id", "key"]
@@ -202,6 +230,8 @@ class ProjectVariablesStream(ProjectBasedStream):
 
 
 class GroupsStream(GroupBasedStream):
+    """Gitlab Groups stream."""
+
     name = "groups"
     path = "/groups/{group_path}"
     primary_keys = ["id"]
@@ -216,19 +246,25 @@ class GroupProjectsStream(GroupBasedStream):
 
 
 class GroupMilestonesStream(GroupBasedStream):
+    """Gitlab Group Milestones stream."""
+
     name = "group_milestones"
     path = "/groups/{group_path}/milestones"
     primary_keys = ["id"]
     schema_filename = "milestones.json"
 
 
 class GroupMembersStream(GroupBasedStream):
+    """Gitlab Group Members stream."""
+
     name = "group_members"
     path = "/groups/{group_path}/members"
     primary_keys = ["group_id", "id"]
 
 
 class GroupLabelsStream(GroupBasedStream):
+    """Gitlab Group Labels stream."""
+
     name = "group_labels"
     path = "/groups/{group_path}/labels"
     primary_keys = ["group_id", "id"]
@@ -273,6 +309,8 @@ def get_url_params(
 
 
 class GroupVariablesStream(GroupBasedStream):
+    """Gitlab Group Variables stream."""
+
     name = "group_variables"
     path = "/groups/{group_path}/variables"
     primary_keys = ["project_id", "key"]
@@ -282,6 +320,8 @@ class GroupVariablesStream(GroupBasedStream):
 
 
 class GlobalSiteUsersStream(GitLabStream):
+    """Gitlab Global Site Users stream."""
+
     name = "site_users"
     path = "/users"
     primary_keys = ["id"]
@@ -301,7 +341,8 @@ class GlobalSiteUsersStream(GitLabStream):
 
 
 # TODO: Failing with:
-# FatalAPIError: 400 Client Error: Bad Request for path: /projects/{project_path}/repository/tags
+# FatalAPIError: 400 Client Error: Bad Request for path:
+# /projects/{project_path}/repository/tags
 # class TagsStream(ProjectBasedStream):
 #     name = "tags"
 #     path = "/projects/{project_path}/repository/tags"

tap_gitlab/tap.py

@@ -3,13 +3,13 @@
 import inspect
 from typing import List
 
-from singer_sdk import Tap, Stream
+from singer_sdk import Stream, Tap
 from singer_sdk import typing as th  # JSON schema typing helpers
 
+from tap_gitlab import streams
 from tap_gitlab.caching import setup_requests_cache
 from tap_gitlab.client import GroupBasedStream
 from tap_gitlab.streams import GitLabStream, ProjectBasedStream
-from tap_gitlab import streams
 
 OPTIN_STREAM_NAMES = [
     "merge_request_commits",

tap_gitlab/tests/test_core.py

@@ -1,24 +1,23 @@
 """Tests standard tap features using the built-in SDK tests library."""
 
 import datetime
+import os
 
 from singer_sdk.testing import get_standard_tap_tests
 
 from tap_gitlab.tap import TapGitLab
 
 SAMPLE_CONFIG = {
-    "start_date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d")
-    # TODO: Initialize minimal tap config
+    "start_date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d"),
+    "private_token": os.getenv("GITLAB_PRIVATE_TOKEN"),
+    "projects": os.getenv("GITLAB_PROJECTS_TO_FETCH", ""),
 }
 
 
 # Run standard built-in tap tests from the SDK:
 def test_standard_tap_tests():
     """Run standard tap tests from the SDK."""
-    tests = get_standard_tap_tests(
-        TapGitLab,
-        config=SAMPLE_CONFIG
-    )
+    tests = get_standard_tap_tests(TapGitLab, config=SAMPLE_CONFIG)
     for test in tests:
         test()
 

tap_gitlab/transforms.py

@@ -1,9 +1,12 @@
+"""Utility functions."""
 from typing import Any, Dict, List
 
 
 def object_array_to_id_array(items: List[dict]) -> List[int]:
+    """Extract id from nested array of objects."""
     return [item["id"] for item in items]
 
 
 def pop_nested_id(record: Dict[str, Any], key: str) -> List[int]:
+    """Extract id from nested owner object and removes the object."""
     return (record.pop("owner", None) or {}).pop("id", None)