diff --git a/tap_csv/filesystem_config.py b/tap_csv/filesystem_config.py new file mode 100644 index 0000000..c522b86 --- /dev/null +++ b/tap_csv/filesystem_config.py @@ -0,0 +1,87 @@ +"""JSON Schema for each filesystem configuration.""" + +from __future__ import annotations + +from singer_sdk import typing as th # JSON schema typing helpers + +FTP = th.Property( + "ftp", + th.ObjectType( + th.Property( + "host", + th.StringType, + required=True, + description="FTP server host", + ), + th.Property( + "port", + th.IntegerType, + default=21, + description="FTP server port", + ), + th.Property( + "username", + th.StringType, + description="FTP username", + ), + th.Property( + "password", + th.StringType, + secret=True, + description="FTP password", + ), + th.Property( + "encoding", + th.StringType, + default="utf-8", + description="FTP server encoding", + ), + ), + description="FTP connection settings", +) + +GITHUB = th.Property( + "github", + th.ObjectType( + th.Property( + "org", + th.StringType, + required=True, + description=("GitHub organization or user where the repository is located"), + ), + th.Property( + "repo", + th.StringType, + required=True, + description="GitHub repository", + ), + th.Property( + "username", + th.StringType, + required=False, + description="GitHub username", + ), + th.Property( + "token", + th.StringType, + required=False, + secret=True, + description="GitHub token", + ), + ), + description="GitHub connection settings", +) + +DROPBOX = th.Property( + "dropbox", + th.ObjectType( + th.Property( + "token", + th.StringType, + secret=True, + required=True, + description="Dropbox token", + ), + ), + description="Dropbox connection settings", +) diff --git a/tap_csv/tap.py b/tap_csv/tap.py index 8cc70be..43abb44 100644 --- a/tap_csv/tap.py +++ b/tap_csv/tap.py @@ -11,7 +11,7 @@ from singer_sdk.helpers._classproperty import classproperty from singer_sdk.helpers.capabilities import TapCapabilities -from tap_csv.client import CSVStream +from . import client, filesystem_config class TapCSV(Tap): @@ -53,87 +53,9 @@ class TapCSV(Tap): "dropbox", ], ), - th.Property( - "ftp", - th.ObjectType( - th.Property( - "host", - th.StringType, - required=True, - description="FTP server host", - ), - th.Property( - "port", - th.IntegerType, - default=21, - description="FTP server port", - ), - th.Property( - "username", - th.StringType, - description="FTP username", - ), - th.Property( - "password", - th.StringType, - secret=True, - description="FTP password", - ), - th.Property( - "encoding", - th.StringType, - default="utf-8", - description="FTP server encoding", - ), - ), - description="FTP connection settings", - ), - th.Property( - "github", - th.ObjectType( - th.Property( - "org", - th.StringType, - required=True, - description=( - "GitHub organization or user where the repository is located" - ), - ), - th.Property( - "repo", - th.StringType, - required=True, - description="GitHub repository", - ), - th.Property( - "username", - th.StringType, - required=False, - description="GitHub username", - ), - th.Property( - "token", - th.StringType, - required=False, - secret=True, - description="GitHub token", - ), - ), - description="GitHub connection settings", - ), - th.Property( - "dropbox", - th.ObjectType( - th.Property( - "token", - th.StringType, - secret=True, - required=True, - description="Dropbox token", - ), - ), - description="Dropbox connection settings", - ), + filesystem_config.FTP, + filesystem_config.GITHUB, + filesystem_config.DROPBOX, th.Property( "csv_files_definition", th.StringType, @@ -189,7 +111,7 @@ def discover_streams(self) -> list[Stream]: raise ConfigValidationError(msg, errors=errors) return [ - CSVStream( + client.CSVStream( tap=self, name=file_config.get("entity"), file_config=file_config, diff --git a/tests/test_client.py b/tests/test_client.py index e14326e..11b80a7 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -4,7 +4,7 @@ import os -from tap_csv.tap import CSVStream, TapCSV +from tap_csv import client, tap def test_get_file_paths_recursively(): @@ -21,8 +21,8 @@ def test_get_file_paths_recursively(): ] } - stream = CSVStream( - tap=TapCSV(config=SAMPLE_CONFIG, catalog={}, state={}), + stream = client.CSVStream( + tap=tap.TapCSV(config=SAMPLE_CONFIG, catalog={}, state={}), name="test_recursive", file_config=SAMPLE_CONFIG.get("files")[0], filesystem="local",