diff --git a/tap_slack/schemas/channels.py b/tap_slack/schemas/channels.py index d9b363d..ce2b6f3 100644 --- a/tap_slack/schemas/channels.py +++ b/tap_slack/schemas/channels.py @@ -6,11 +6,11 @@ th.Property("is_channel", th.BooleanType), th.Property("is_group", th.BooleanType), th.Property("is_im", th.BooleanType), - th.Property("created", th.StringType), + th.Property("created", th.IntegerType), th.Property("creator", th.StringType), th.Property("is_archived", th.BooleanType), th.Property("is_general", th.BooleanType), - th.Property("unlinked", th.StringType), + th.Property("unlinked", th.IntegerType), th.Property("name_normalized", th.StringType), th.Property("is_shared", th.BooleanType), th.Property("is_ext_shared", th.BooleanType), @@ -28,7 +28,7 @@ th.ObjectType( th.Property("value", th.StringType), th.Property("creator", th.StringType), - th.Property("last_set", th.StringType), + th.Property("last_set", th.IntegerType), ), ), th.Property( @@ -36,7 +36,7 @@ th.ObjectType( th.Property("value", th.StringType), th.Property("creator", th.StringType), - th.Property("last_set", th.StringType), + th.Property("last_set", th.IntegerType), ), ), th.Property("previous_names", th.ArrayType(th.StringType)), diff --git a/tap_slack/schemas/messages.py b/tap_slack/schemas/messages.py index 42bb1cb..a9cd1af 100644 --- a/tap_slack/schemas/messages.py +++ b/tap_slack/schemas/messages.py @@ -4,7 +4,7 @@ th.Property("channel_id", th.StringType, required=True), th.Property( "ts", - th.NumberType, + th.StringType, required=True, description="Epoch timestamp of when the thread reply was posted.", ), @@ -25,7 +25,7 @@ th.Property("id", th.StringType), th.Property("name", th.StringType), th.Property("team_id", th.StringType), - th.Property("updated", th.StringType), + th.Property("updated", th.IntegerType), ), ), th.Property("client_msg_id", th.StringType), diff --git a/tap_slack/streams.py b/tap_slack/streams.py index d914299..afd00e4 100644 --- a/tap_slack/streams.py +++ b/tap_slack/streams.py @@ -32,9 +32,21 @@ def get_url_params(self, context, next_page_token): def post_process(self, row, context): "Join the channel if not a member, but emit no data." row = super().post_process(row, context) - if not row["is_member"]: - self._join_channel(row["id"]) - return row + # return all in selected_channels or default to all, exclude any in excluded_channels list + channel_id = row["id"] + if self._is_channel_included(channel_id): + if not row["is_member"] and self.config.get("auto_join_channels", False): + self._join_channel(channel_id) + return row + + def _is_channel_included(self, channel_id: str) -> bool: + selected_channels = self.config.get("selected_channels") + excluded_channels = self.config.get("excluded_channels", []) + if channel_id in excluded_channels: + return False + if selected_channels and channel_id not in selected_channels: + return False + return True def _join_channel(self, channel_id: str) -> requests.Response: url = f"{self.url_base}/conversations.join" diff --git a/tap_slack/tap.py b/tap_slack/tap.py index 7307ea6..0a65b89 100644 --- a/tap_slack/tap.py +++ b/tap_slack/tap.py @@ -58,6 +58,16 @@ class TapSlack(Tap): default=False, description="Whether the bot user should attempt to join channels that it has not yet joined. The bot user must be a member of the channel to retrieve messages.", ), + th.Property( + "selected_channels", + th.ArrayType(th.StringType), + description="A list of channel IDs that should be retrieved. If not defined then all are selected.", + ), + th.Property( + "excluded_channels", + th.ArrayType(th.StringType), + description="A list of channel IDs that should not be retrieved. Excluding overrides a selected setting, so if a channel is included in both selected and excluded, it will be excluded.", + ), ).to_dict() def discover_streams(self) -> List[Stream]: @@ -71,3 +81,6 @@ def expectations(self): "tap__discovery", "tap__stream_connections", ] + +if __name__ == "__main__": + TapSlack.cli() \ No newline at end of file diff --git a/tests/test_core.py b/tests/test_core.py index 6fedf57..82c3c37 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -2,6 +2,7 @@ import pytest import os +from unittest.mock import patch from tap_slack.tap import TapSlack from tap_slack.testing import TapTestUtility @@ -29,3 +30,17 @@ def test_builtin_tap_tests(test_util, test_config): test_name, params = test_config test_func = test_util.available_tests[test_name] test_func(**params) + + +@patch("tap_slack.streams.ChannelsStream._join_channel") +def test_auto_join_channel_false(patch_obj, test_util): + test_utility.run_sync() + patch_obj.assert_not_called() + +@patch("tap_slack.streams.ChannelsStream._join_channel") +def test_auto_join_channel(patch_obj, test_util): + config = SAMPLE_CONFIG.copy() + config["auto_join_channels"] = True + test_utility = TapTestUtility(TapSlack, config, stream_record_limit=500) + test_utility.run_sync() + patch_obj.assert_called()