Merge pull request #7 from pnadolny13/auto_join_fix_and_channels_filter

Type mismatches, `auto_join_channel` implemented, and add channels filter

tap_slack/schemas/channels.py

@@ -6,11 +6,11 @@
     th.Property("is_channel", th.BooleanType),
     th.Property("is_group", th.BooleanType),
     th.Property("is_im", th.BooleanType),
-    th.Property("created", th.StringType),
+    th.Property("created", th.IntegerType),
     th.Property("creator", th.StringType),
     th.Property("is_archived", th.BooleanType),
     th.Property("is_general", th.BooleanType),
-    th.Property("unlinked", th.StringType),
+    th.Property("unlinked", th.IntegerType),
     th.Property("name_normalized", th.StringType),
     th.Property("is_shared", th.BooleanType),
     th.Property("is_ext_shared", th.BooleanType),
@@ -28,15 +28,15 @@
         th.ObjectType(
             th.Property("value", th.StringType),
             th.Property("creator", th.StringType),
-            th.Property("last_set", th.StringType),
+            th.Property("last_set", th.IntegerType),
         ),
     ),
     th.Property(
         "purpose",
         th.ObjectType(
             th.Property("value", th.StringType),
             th.Property("creator", th.StringType),
-            th.Property("last_set", th.StringType),
+            th.Property("last_set", th.IntegerType),
         ),
     ),
     th.Property("previous_names", th.ArrayType(th.StringType)),

tap_slack/schemas/messages.py

@@ -4,7 +4,7 @@
     th.Property("channel_id", th.StringType, required=True),
     th.Property(
         "ts",
-        th.NumberType,
+        th.StringType,
         required=True,
         description="Epoch timestamp of when the thread reply was posted.",
     ),
@@ -25,7 +25,7 @@
             th.Property("id", th.StringType),
             th.Property("name", th.StringType),
             th.Property("team_id", th.StringType),
-            th.Property("updated", th.StringType),
+            th.Property("updated", th.IntegerType),
         ),
     ),
     th.Property("client_msg_id", th.StringType),

tap_slack/streams.py

@@ -32,9 +32,21 @@ def get_url_params(self, context, next_page_token):
     def post_process(self, row, context):
         "Join the channel if not a member, but emit no data."
         row = super().post_process(row, context)
-        if not row["is_member"]:
-            self._join_channel(row["id"])
-        return row
+        # return all in selected_channels or default to all, exclude any in excluded_channels list
+        channel_id = row["id"]
+        if self._is_channel_included(channel_id):
+            if not row["is_member"] and self.config.get("auto_join_channels", False):
+                self._join_channel(channel_id)
+            return row
+
+    def _is_channel_included(self, channel_id: str) -> bool:
+        selected_channels = self.config.get("selected_channels")
+        excluded_channels = self.config.get("excluded_channels", [])
+        if channel_id in excluded_channels:
+                return False
+        if selected_channels and channel_id not in selected_channels:
+                return False
+        return True
 
     def _join_channel(self, channel_id: str) -> requests.Response:
         url = f"{self.url_base}/conversations.join"

tap_slack/tap.py

@@ -58,6 +58,16 @@ class TapSlack(Tap):
             default=False,
             description="Whether the bot user should attempt to join channels that it has not yet joined. The bot user must be a member of the channel to retrieve messages.",
         ),
+        th.Property(
+            "selected_channels",
+            th.ArrayType(th.StringType),
+            description="A list of channel IDs that should be retrieved. If not defined then all are selected.",
+        ),
+        th.Property(
+            "excluded_channels",
+            th.ArrayType(th.StringType),
+            description="A list of channel IDs that should not be retrieved. Excluding overrides a selected setting, so if a channel is included in both selected and excluded, it will be excluded.",
+        ),
     ).to_dict()
 
     def discover_streams(self) -> List[Stream]:
@@ -71,3 +81,6 @@ def expectations(self):
             "tap__discovery",
             "tap__stream_connections",
         ]
+
+if __name__ == "__main__":
+    TapSlack.cli()

tests/test_core.py

@@ -2,6 +2,7 @@
 
 import pytest
 import os
+from unittest.mock import patch
 
 from tap_slack.tap import TapSlack
 from tap_slack.testing import TapTestUtility
@@ -29,3 +30,17 @@ def test_builtin_tap_tests(test_util, test_config):
     test_name, params = test_config
     test_func = test_util.available_tests[test_name]
     test_func(**params)
+
+
+@patch("tap_slack.streams.ChannelsStream._join_channel")
+def test_auto_join_channel_false(patch_obj, test_util):
+    test_utility.run_sync()
+    patch_obj.assert_not_called()
+
+@patch("tap_slack.streams.ChannelsStream._join_channel")
+def test_auto_join_channel(patch_obj, test_util):
+    config = SAMPLE_CONFIG.copy()
+    config["auto_join_channels"] = True
+    test_utility = TapTestUtility(TapSlack, config, stream_record_limit=500)
+    test_utility.run_sync()
+    patch_obj.assert_called()