Move FS configs to their own module

MeltanoLabs · Sep 6, 2024 · 09d3256 · 09d3256
1 parent 2f45921
commit 09d3256
Show file tree

Hide file tree

Showing 3 changed files with 95 additions and 86 deletions.
diff --git a/tap_csv/filesystem_config.py b/tap_csv/filesystem_config.py
@@ -0,0 +1,87 @@
+"""JSON Schema for each filesystem configuration."""
+
+from __future__ import annotations
+
+from singer_sdk import typing as th  # JSON schema typing helpers
+
+FTP = th.Property(
+    "ftp",
+    th.ObjectType(
+        th.Property(
+            "host",
+            th.StringType,
+            required=True,
+            description="FTP server host",
+        ),
+        th.Property(
+            "port",
+            th.IntegerType,
+            default=21,
+            description="FTP server port",
+        ),
+        th.Property(
+            "username",
+            th.StringType,
+            description="FTP username",
+        ),
+        th.Property(
+            "password",
+            th.StringType,
+            secret=True,
+            description="FTP password",
+        ),
+        th.Property(
+            "encoding",
+            th.StringType,
+            default="utf-8",
+            description="FTP server encoding",
+        ),
+    ),
+    description="FTP connection settings",
+)
+
+GITHUB = th.Property(
+    "github",
+    th.ObjectType(
+        th.Property(
+            "org",
+            th.StringType,
+            required=True,
+            description=("GitHub organization or user where the repository is located"),
+        ),
+        th.Property(
+            "repo",
+            th.StringType,
+            required=True,
+            description="GitHub repository",
+        ),
+        th.Property(
+            "username",
+            th.StringType,
+            required=False,
+            description="GitHub username",
+        ),
+        th.Property(
+            "token",
+            th.StringType,
+            required=False,
+            secret=True,
+            description="GitHub token",
+        ),
+    ),
+    description="GitHub connection settings",
+)
+
+DROPBOX = th.Property(
+    "dropbox",
+    th.ObjectType(
+        th.Property(
+            "token",
+            th.StringType,
+            secret=True,
+            required=True,
+            description="Dropbox token",
+        ),
+    ),
+    description="Dropbox connection settings",
+)
diff --git a/tap_csv/tap.py b/tap_csv/tap.py
@@ -11,7 +11,7 @@
 from singer_sdk.helpers._classproperty import classproperty
 from singer_sdk.helpers.capabilities import TapCapabilities
 
-from tap_csv.client import CSVStream
+from . import client, filesystem_config
 
 
 class TapCSV(Tap):
@@ -53,87 +53,9 @@ class TapCSV(Tap):
                 "dropbox",
             ],
         ),
-        th.Property(
-            "ftp",
-            th.ObjectType(
-                th.Property(
-                    "host",
-                    th.StringType,
-                    required=True,
-                    description="FTP server host",
-                ),
-                th.Property(
-                    "port",
-                    th.IntegerType,
-                    default=21,
-                    description="FTP server port",
-                ),
-                th.Property(
-                    "username",
-                    th.StringType,
-                    description="FTP username",
-                ),
-                th.Property(
-                    "password",
-                    th.StringType,
-                    secret=True,
-                    description="FTP password",
-                ),
-                th.Property(
-                    "encoding",
-                    th.StringType,
-                    default="utf-8",
-                    description="FTP server encoding",
-                ),
-            ),
-            description="FTP connection settings",
-        ),
-        th.Property(
-            "github",
-            th.ObjectType(
-                th.Property(
-                    "org",
-                    th.StringType,
-                    required=True,
-                    description=(
-                        "GitHub organization or user where the repository is located"
-                    ),
-                ),
-                th.Property(
-                    "repo",
-                    th.StringType,
-                    required=True,
-                    description="GitHub repository",
-                ),
-                th.Property(
-                    "username",
-                    th.StringType,
-                    required=False,
-                    description="GitHub username",
-                ),
-                th.Property(
-                    "token",
-                    th.StringType,
-                    required=False,
-                    secret=True,
-                    description="GitHub token",
-                ),
-            ),
-            description="GitHub connection settings",
-        ),
-        th.Property(
-            "dropbox",
-            th.ObjectType(
-                th.Property(
-                    "token",
-                    th.StringType,
-                    secret=True,
-                    required=True,
-                    description="Dropbox token",
-                ),
-            ),
-            description="Dropbox connection settings",
-        ),
+        filesystem_config.FTP,
+        filesystem_config.GITHUB,
+        filesystem_config.DROPBOX,
         th.Property(
             "csv_files_definition",
             th.StringType,
@@ -189,7 +111,7 @@ def discover_streams(self) -> list[Stream]:
             raise ConfigValidationError(msg, errors=errors)
 
         return [
-            CSVStream(
+            client.CSVStream(
                 tap=self,
                 name=file_config.get("entity"),
                 file_config=file_config,

diff --git a/tests/test_client.py b/tests/test_client.py
@@ -4,7 +4,7 @@
 
 import os
 
-from tap_csv.tap import CSVStream, TapCSV
+from tap_csv import client, tap
 
 
 def test_get_file_paths_recursively():
@@ -21,8 +21,8 @@ def test_get_file_paths_recursively():
         ]
     }
 
-    stream = CSVStream(
-        tap=TapCSV(config=SAMPLE_CONFIG, catalog={}, state={}),
+    stream = client.CSVStream(
+        tap=tap.TapCSV(config=SAMPLE_CONFIG, catalog={}, state={}),
         name="test_recursive",
         file_config=SAMPLE_CONFIG.get("files")[0],
         filesystem="local",