From b815f7c25eea898506bbd807824eca8d3c110271 Mon Sep 17 00:00:00 2001 From: pnadolny13 Date: Mon, 6 Dec 2021 12:02:43 -0500 Subject: [PATCH 01/12] implement autojoin logic and add a channels filter --- tap_slack/streams.py | 10 +++++++--- tap_slack/tap.py | 8 ++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/tap_slack/streams.py b/tap_slack/streams.py index d914299..3a2d245 100644 --- a/tap_slack/streams.py +++ b/tap_slack/streams.py @@ -32,9 +32,13 @@ def get_url_params(self, context, next_page_token): def post_process(self, row, context): "Join the channel if not a member, but emit no data." row = super().post_process(row, context) - if not row["is_member"]: - self._join_channel(row["id"]) - return row + # only return selected channels and default to all + channels = self.config.get("channels") + if (not channels or row["id"] in channels): + if not row["is_member"]: + if self.config.get("auto_join_channels", False): + self._join_channel(row["id"]) + return row def _join_channel(self, channel_id: str) -> requests.Response: url = f"{self.url_base}/conversations.join" diff --git a/tap_slack/tap.py b/tap_slack/tap.py index 7307ea6..1331438 100644 --- a/tap_slack/tap.py +++ b/tap_slack/tap.py @@ -58,6 +58,11 @@ class TapSlack(Tap): default=False, description="Whether the bot user should attempt to join channels that it has not yet joined. The bot user must be a member of the channel to retrieve messages.", ), + th.Property( + "channels", + th.ArrayType(th.StringType), + description="A list of channel IDs that should be retrieved. If not defined then all are selected.", + ), ).to_dict() def discover_streams(self) -> List[Stream]: @@ -71,3 +76,6 @@ def expectations(self): "tap__discovery", "tap__stream_connections", ] + +if __name__ == "__main__": + TapSlack.cli() \ No newline at end of file From 81a6adcd9094d232a6c30c9205f7b92fb347e331 Mon Sep 17 00:00:00 2001 From: pnadolny13 Date: Mon, 6 Dec 2021 12:12:50 -0500 Subject: [PATCH 02/12] fix schema mistyped fields --- tap_slack/schemas/channels.py | 8 ++++---- tap_slack/schemas/messages.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tap_slack/schemas/channels.py b/tap_slack/schemas/channels.py index d9b363d..c523a54 100644 --- a/tap_slack/schemas/channels.py +++ b/tap_slack/schemas/channels.py @@ -7,10 +7,10 @@ th.Property("is_group", th.BooleanType), th.Property("is_im", th.BooleanType), th.Property("created", th.StringType), - th.Property("creator", th.StringType), + th.Property("creator", th.IntegerType), th.Property("is_archived", th.BooleanType), th.Property("is_general", th.BooleanType), - th.Property("unlinked", th.StringType), + th.Property("unlinked", th.IntegerType), th.Property("name_normalized", th.StringType), th.Property("is_shared", th.BooleanType), th.Property("is_ext_shared", th.BooleanType), @@ -28,7 +28,7 @@ th.ObjectType( th.Property("value", th.StringType), th.Property("creator", th.StringType), - th.Property("last_set", th.StringType), + th.Property("last_set", th.IntegerType), ), ), th.Property( @@ -36,7 +36,7 @@ th.ObjectType( th.Property("value", th.StringType), th.Property("creator", th.StringType), - th.Property("last_set", th.StringType), + th.Property("last_set", th.IntegerType), ), ), th.Property("previous_names", th.ArrayType(th.StringType)), diff --git a/tap_slack/schemas/messages.py b/tap_slack/schemas/messages.py index 42bb1cb..78caf87 100644 --- a/tap_slack/schemas/messages.py +++ b/tap_slack/schemas/messages.py @@ -4,7 +4,7 @@ th.Property("channel_id", th.StringType, required=True), th.Property( "ts", - th.NumberType, + th.StringType, required=True, description="Epoch timestamp of when the thread reply was posted.", ), From ea395e1e78c871323337f48e13333c50cd8e9caf Mon Sep 17 00:00:00 2001 From: pnadolny13 Date: Mon, 6 Dec 2021 16:08:39 -0500 Subject: [PATCH 03/12] updated field is an integer not a string --- tap_slack/schemas/messages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_slack/schemas/messages.py b/tap_slack/schemas/messages.py index 78caf87..a9cd1af 100644 --- a/tap_slack/schemas/messages.py +++ b/tap_slack/schemas/messages.py @@ -25,7 +25,7 @@ th.Property("id", th.StringType), th.Property("name", th.StringType), th.Property("team_id", th.StringType), - th.Property("updated", th.StringType), + th.Property("updated", th.IntegerType), ), ), th.Property("client_msg_id", th.StringType), From e45b125310989b083f0071a278d48c1c375fb53a Mon Sep 17 00:00:00 2001 From: pnadolny13 Date: Fri, 10 Dec 2021 20:53:35 -0500 Subject: [PATCH 04/12] add auto join test --- tap_slack/streams.py | 7 ++++--- tests/test_core.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/tap_slack/streams.py b/tap_slack/streams.py index 3a2d245..7e3d12d 100644 --- a/tap_slack/streams.py +++ b/tap_slack/streams.py @@ -33,11 +33,12 @@ def post_process(self, row, context): "Join the channel if not a member, but emit no data." row = super().post_process(row, context) # only return selected channels and default to all - channels = self.config.get("channels") - if (not channels or row["id"] in channels): + selected_channels = self.config.get("channels") + channel_name = row["id"] + if not selected_channels or channel_name in selected_channels: if not row["is_member"]: if self.config.get("auto_join_channels", False): - self._join_channel(row["id"]) + self._join_channel(channel_name) return row def _join_channel(self, channel_id: str) -> requests.Response: diff --git a/tests/test_core.py b/tests/test_core.py index 6fedf57..82c3c37 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -2,6 +2,7 @@ import pytest import os +from unittest.mock import patch from tap_slack.tap import TapSlack from tap_slack.testing import TapTestUtility @@ -29,3 +30,17 @@ def test_builtin_tap_tests(test_util, test_config): test_name, params = test_config test_func = test_util.available_tests[test_name] test_func(**params) + + +@patch("tap_slack.streams.ChannelsStream._join_channel") +def test_auto_join_channel_false(patch_obj, test_util): + test_utility.run_sync() + patch_obj.assert_not_called() + +@patch("tap_slack.streams.ChannelsStream._join_channel") +def test_auto_join_channel(patch_obj, test_util): + config = SAMPLE_CONFIG.copy() + config["auto_join_channels"] = True + test_utility = TapTestUtility(TapSlack, config, stream_record_limit=500) + test_utility.run_sync() + patch_obj.assert_called() From b8d0cad39e1f6d116abbe74c21df10eea64c789e Mon Sep 17 00:00:00 2001 From: pnadolny13 Date: Fri, 10 Dec 2021 20:59:59 -0500 Subject: [PATCH 05/12] update channel creator type --- tap_slack/schemas/channels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_slack/schemas/channels.py b/tap_slack/schemas/channels.py index c523a54..92d18a6 100644 --- a/tap_slack/schemas/channels.py +++ b/tap_slack/schemas/channels.py @@ -7,7 +7,7 @@ th.Property("is_group", th.BooleanType), th.Property("is_im", th.BooleanType), th.Property("created", th.StringType), - th.Property("creator", th.IntegerType), + th.Property("creator", th.StringType), th.Property("is_archived", th.BooleanType), th.Property("is_general", th.BooleanType), th.Property("unlinked", th.IntegerType), From 62949fd3638ab5017632f699350a7fc760014b62 Mon Sep 17 00:00:00 2001 From: pnadolny13 Date: Fri, 10 Dec 2021 21:10:36 -0500 Subject: [PATCH 06/12] fix channel created type --- tap_slack/schemas/channels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_slack/schemas/channels.py b/tap_slack/schemas/channels.py index 92d18a6..ce2b6f3 100644 --- a/tap_slack/schemas/channels.py +++ b/tap_slack/schemas/channels.py @@ -6,7 +6,7 @@ th.Property("is_channel", th.BooleanType), th.Property("is_group", th.BooleanType), th.Property("is_im", th.BooleanType), - th.Property("created", th.StringType), + th.Property("created", th.IntegerType), th.Property("creator", th.StringType), th.Property("is_archived", th.BooleanType), th.Property("is_general", th.BooleanType), From 18e177f505c9eb424a54f073fa1401a00b39a909 Mon Sep 17 00:00:00 2001 From: pnadolny13 Date: Fri, 10 Dec 2021 21:20:26 -0500 Subject: [PATCH 07/12] cleanup code --- tap_slack/streams.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tap_slack/streams.py b/tap_slack/streams.py index 7e3d12d..e2168f1 100644 --- a/tap_slack/streams.py +++ b/tap_slack/streams.py @@ -33,14 +33,18 @@ def post_process(self, row, context): "Join the channel if not a member, but emit no data." row = super().post_process(row, context) # only return selected channels and default to all - selected_channels = self.config.get("channels") channel_name = row["id"] - if not selected_channels or channel_name in selected_channels: - if not row["is_member"]: - if self.config.get("auto_join_channels", False): - self._join_channel(channel_name) + if self._do_sync_channel(channel_name): + if not row["is_member"] and self.config.get("auto_join_channels", False): + self._join_channel(channel_name) return row + def _do_sync_channel(self, channel_name: str) -> bool: + selected_channels = self.config.get("channels") + if not selected_channels or channel_name in selected_channels: + return True + return False + def _join_channel(self, channel_id: str) -> requests.Response: url = f"{self.url_base}/conversations.join" params = {"channel": channel_id} From 008a94da46eb3b38d9af396d9f3ee94e44855ee6 Mon Sep 17 00:00:00 2001 From: pnadolny13 Date: Mon, 13 Dec 2021 20:37:44 -0500 Subject: [PATCH 08/12] add selected_channels and excluded_channels --- tap_slack/streams.py | 16 ++++++++++++---- tap_slack/tap.py | 7 ++++++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/tap_slack/streams.py b/tap_slack/streams.py index e2168f1..4931bf1 100644 --- a/tap_slack/streams.py +++ b/tap_slack/streams.py @@ -34,16 +34,24 @@ def post_process(self, row, context): row = super().post_process(row, context) # only return selected channels and default to all channel_name = row["id"] - if self._do_sync_channel(channel_name): + if self._is_included(channel_name) and not self._is_excluded(channel_name): if not row["is_member"] and self.config.get("auto_join_channels", False): self._join_channel(channel_name) return row - def _do_sync_channel(self, channel_name: str) -> bool: - selected_channels = self.config.get("channels") + def _is_excluded(self, channel_name: str) -> bool: + excluded_channels = self.config.get("excluded_channels") + if channel_name in excluded_channels: + return True + else: + return False + + def _is_included(self, channel_name: str) -> bool: + selected_channels = self.config.get("selected_channels") if not selected_channels or channel_name in selected_channels: return True - return False + else: + return False def _join_channel(self, channel_id: str) -> requests.Response: url = f"{self.url_base}/conversations.join" diff --git a/tap_slack/tap.py b/tap_slack/tap.py index 1331438..0a65b89 100644 --- a/tap_slack/tap.py +++ b/tap_slack/tap.py @@ -59,10 +59,15 @@ class TapSlack(Tap): description="Whether the bot user should attempt to join channels that it has not yet joined. The bot user must be a member of the channel to retrieve messages.", ), th.Property( - "channels", + "selected_channels", th.ArrayType(th.StringType), description="A list of channel IDs that should be retrieved. If not defined then all are selected.", ), + th.Property( + "excluded_channels", + th.ArrayType(th.StringType), + description="A list of channel IDs that should not be retrieved. Excluding overrides a selected setting, so if a channel is included in both selected and excluded, it will be excluded.", + ), ).to_dict() def discover_streams(self) -> List[Stream]: From 9f528d0624521f4bf0cf0ec524d2e95b7170d8fa Mon Sep 17 00:00:00 2001 From: pnadolny13 Date: Mon, 13 Dec 2021 20:40:06 -0500 Subject: [PATCH 09/12] default exclude list --- tap_slack/streams.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tap_slack/streams.py b/tap_slack/streams.py index 4931bf1..4056ee4 100644 --- a/tap_slack/streams.py +++ b/tap_slack/streams.py @@ -32,7 +32,7 @@ def get_url_params(self, context, next_page_token): def post_process(self, row, context): "Join the channel if not a member, but emit no data." row = super().post_process(row, context) - # only return selected channels and default to all + # return all in selected_channels or default to all, exclude any in excluded_channels list channel_name = row["id"] if self._is_included(channel_name) and not self._is_excluded(channel_name): if not row["is_member"] and self.config.get("auto_join_channels", False): @@ -40,7 +40,7 @@ def post_process(self, row, context): return row def _is_excluded(self, channel_name: str) -> bool: - excluded_channels = self.config.get("excluded_channels") + excluded_channels = self.config.get("excluded_channels", []) if channel_name in excluded_channels: return True else: From ee9a647dd8d69f46d3f77dea63c98ae99d2bb806 Mon Sep 17 00:00:00 2001 From: pnadolny13 Date: Tue, 14 Dec 2021 08:16:06 -0500 Subject: [PATCH 10/12] rename to channel_id and add channel to function names for clarity --- tap_slack/streams.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tap_slack/streams.py b/tap_slack/streams.py index 4056ee4..ee82e8b 100644 --- a/tap_slack/streams.py +++ b/tap_slack/streams.py @@ -33,22 +33,22 @@ def post_process(self, row, context): "Join the channel if not a member, but emit no data." row = super().post_process(row, context) # return all in selected_channels or default to all, exclude any in excluded_channels list - channel_name = row["id"] - if self._is_included(channel_name) and not self._is_excluded(channel_name): + channel_id = row["id"] + if self._is_channel_inluded(channel_id) and not self._is_channel_excluded(channel_id): if not row["is_member"] and self.config.get("auto_join_channels", False): - self._join_channel(channel_name) + self._join_channel(channel_id) return row - def _is_excluded(self, channel_name: str) -> bool: + def _is_channel_excluded(self, channel_id: str) -> bool: excluded_channels = self.config.get("excluded_channels", []) - if channel_name in excluded_channels: + if channel_id in excluded_channels: return True else: return False - def _is_included(self, channel_name: str) -> bool: + def _is_channel_inluded(self, channel_id: str) -> bool: selected_channels = self.config.get("selected_channels") - if not selected_channels or channel_name in selected_channels: + if not selected_channels or channel_id in selected_channels: return True else: return False From f286447faeece88f3ba1182b73e237e3c8ac3b0d Mon Sep 17 00:00:00 2001 From: pnadolny13 Date: Tue, 14 Dec 2021 08:20:19 -0500 Subject: [PATCH 11/12] refactor to single _is_channel_included method --- tap_slack/streams.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/tap_slack/streams.py b/tap_slack/streams.py index ee82e8b..018cd21 100644 --- a/tap_slack/streams.py +++ b/tap_slack/streams.py @@ -34,22 +34,19 @@ def post_process(self, row, context): row = super().post_process(row, context) # return all in selected_channels or default to all, exclude any in excluded_channels list channel_id = row["id"] - if self._is_channel_inluded(channel_id) and not self._is_channel_excluded(channel_id): + if self._is_channel_included(channel_id): if not row["is_member"] and self.config.get("auto_join_channels", False): self._join_channel(channel_id) return row - def _is_channel_excluded(self, channel_id: str) -> bool: - excluded_channels = self.config.get("excluded_channels", []) - if channel_id in excluded_channels: - return True - else: - return False - - def _is_channel_inluded(self, channel_id: str) -> bool: + def _is_channel_included(self, channel_id: str) -> bool: selected_channels = self.config.get("selected_channels") if not selected_channels or channel_id in selected_channels: - return True + excluded_channels = self.config.get("excluded_channels", []) + if channel_id in excluded_channels: + return False + else: + return True else: return False From 1291d6149ac504e45c9c85d60f2cd48a42ef7861 Mon Sep 17 00:00:00 2001 From: pnadolny13 Date: Tue, 14 Dec 2021 09:39:33 -0500 Subject: [PATCH 12/12] _is_channel_included refactor --- tap_slack/streams.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tap_slack/streams.py b/tap_slack/streams.py index 018cd21..afd00e4 100644 --- a/tap_slack/streams.py +++ b/tap_slack/streams.py @@ -41,14 +41,12 @@ def post_process(self, row, context): def _is_channel_included(self, channel_id: str) -> bool: selected_channels = self.config.get("selected_channels") - if not selected_channels or channel_id in selected_channels: - excluded_channels = self.config.get("excluded_channels", []) - if channel_id in excluded_channels: + excluded_channels = self.config.get("excluded_channels", []) + if channel_id in excluded_channels: return False - else: - return True - else: - return False + if selected_channels and channel_id not in selected_channels: + return False + return True def _join_channel(self, channel_id: str) -> requests.Response: url = f"{self.url_base}/conversations.join"