From a4155deb82de1b64ed24acd7d02f73de9be2e1d4 Mon Sep 17 00:00:00 2001 From: Animesh Kumar Date: Sat, 2 May 2020 22:31:21 +0530 Subject: [PATCH] [backend] Add Rocket.Chat backend This commit adds support to fetch messages from a Rocket.Chat channel. The tests have been added accordingly. The usage docs have been updated. Release notes have also been added. Signed-off-by: Animesh Kumar --- README.md | 8 + bin/perceval | 1 + perceval/backends/core/rocketchat.py | 404 +++++++++++++ .../unreleased/add-rocket.chat-backend.yml | 13 + tests/data/rocketchat/channel_info.json | 38 ++ .../rocketchat/message_empty_2020_05_10.json | 7 + tests/data/rocketchat/message_page_1.json | 62 ++ tests/data/rocketchat/message_page_2.json | 43 ++ .../rocketchat/message_page_2020_05_03.json | 22 + tests/test_rocketchat.py | 564 ++++++++++++++++++ 10 files changed, 1162 insertions(+) create mode 100644 perceval/backends/core/rocketchat.py create mode 100644 releases/unreleased/add-rocket.chat-backend.yml create mode 100644 tests/data/rocketchat/channel_info.json create mode 100644 tests/data/rocketchat/message_empty_2020_05_10.json create mode 100644 tests/data/rocketchat/message_page_1.json create mode 100644 tests/data/rocketchat/message_page_2.json create mode 100644 tests/data/rocketchat/message_page_2020_05_03.json create mode 100644 tests/test_rocketchat.py diff --git a/README.md b/README.md index e905ef302..23b0057d0 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ are: phabricator Fetch tasks from a Phabricator site pipermail Fetch messages from a Pipermail archiver redmine Fetch issues from a Redmine server + rocketchat Fetch messages from a Rocket.Chat channel rss Fetch entries from a RSS feed server slack Fetch messages from a Slack channel stackexchange Fetch questions from StackExchange sites @@ -321,6 +322,13 @@ $ perceval pipermail 'http://mail-archives.apache.org/mod_mbox/httpd-dev/' $ perceval redmine 'https://www.redmine.org/' --from-date '2016-01-01' -t abcdefghijk ``` +### Rocket.Chat + +Rocket.Chat backend needs an API token and a User Id to authenticate to the server. +``` +$ perceval rocketchat -t 'abchdefghij' -u '1234abcd' --from-date '2020-05-02' https://open.rocket.chat general +``` + ### RSS ``` $ perceval rss 'https://blog.bitergia.com/feed/' diff --git a/bin/perceval b/bin/perceval index bf223ddf1..98e9344ba 100755 --- a/bin/perceval +++ b/bin/perceval @@ -73,6 +73,7 @@ are: phabricator Fetch tasks from a Phabricator site pipermail Fetch messages from a Pipermail archiver redmine Fetch issues from a Redmine server + rocketchat Fetch messages from a Rocket.Chat channel rss Fetch entries from a RSS feed server slack Fetch messages from a Slack channel stackexchange Fetch questions from StackExchange sites diff --git a/perceval/backends/core/rocketchat.py b/perceval/backends/core/rocketchat.py new file mode 100644 index 000000000..f1c890e3c --- /dev/null +++ b/perceval/backends/core/rocketchat.py @@ -0,0 +1,404 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2015-2020 Bitergia +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Authors: +# Aditya Prajapati +# Animesh Kumar +# + +import logging +import json + +from grimoirelab_toolkit.uris import urijoin +from grimoirelab_toolkit.datetime import (datetime_utcnow, + datetime_to_utc, + str_to_datetime) + +from ...backend import (Backend, + BackendCommand, + BackendCommandArgumentParser) +from ...client import HttpClient, RateLimitHandler +from ...utils import DEFAULT_DATETIME + +CATEGORY_MESSAGE = "message" + +API_EXTENSION = "/api/v1/" + +MIN_RATE_LIMIT = 10 +MAX_ITEMS = 100 + +logger = logging.getLogger(__name__) + + +class RocketChat(Backend): + """Rocket.Chat backend. + + This class allows to fetch messages from a channel(room) on a Rocket.Chat server. + An API token and a User Id is required to access the server. + + :param url: server url from where messages are to be fetched + :param channel: name of the channel from where data will be fetched + :param user_id: generated User Id using your Rocket.Chat account + :param api_token: token needed to use the API + :param max_items: maximum number of message requested on the same query + :param sleep_for_rate: sleep until rate limit is reset + :param min_rate_to_sleep: minimum rate needed to sleep until + it will be reset + :param tag: label used to mark the data + :param archive: archive to store/retrieve items + :param ssl_verify: enable/disable SSL verification + """ + version = '0.1.0' + + CATEGORIES = [CATEGORY_MESSAGE] + EXTRA_SEARCH_FIELDS = { + 'channel_name': ['channel_info', 'name'], + 'channel_id': ['channel_info', '_id'] + } + + def __init__(self, url, channel, user_id, api_token, max_items=MAX_ITEMS, + sleep_for_rate=False, min_rate_to_sleep=MIN_RATE_LIMIT, + tag=None, archive=None, ssl_verify=True): + origin = urijoin(url, channel) + + super().__init__(origin, tag=tag, archive=archive, ssl_verify=ssl_verify) + + self.url = url + self.channel = channel + self.user_id = user_id + self.api_token = api_token + self.max_items = max_items + self.sleep_for_rate = sleep_for_rate + self.min_rate_to_sleep = min_rate_to_sleep + self.client = None + + def fetch(self, category=CATEGORY_MESSAGE, from_date=DEFAULT_DATETIME, filter_classified=False): + """Fetch the messages from the channel. + + This method fetches the messages stored on the channel that were + sent since the given date. + + :param category: the category of items to fetch + :param from_date: obtain messages sent since this date + :param filter_classified: remove classified fields from the resulting items + + :returns: a generator of messages + """ + if not from_date: + from_date = DEFAULT_DATETIME + from_date = datetime_to_utc(from_date) + + kwargs = {'from_date': from_date} + + items = super().fetch(category, **kwargs) + + return items + + def fetch_items(self, category, **kwargs): + """Fetch the messages. + + :param category: the category of items to fetch + :param kwargs: backend arguments + + :returns: a generator of items + """ + from_date = kwargs['from_date'] + logger.info("Fetching messages of channel: %s from date: %s", + self.channel, from_date) + + raw_channel_info = self.client.channel_info(self.channel) + channel_info = self.parse_channel_info(raw_channel_info) + + fetching = True + nmsgs = 0 + offset = 0 + + while fetching: + raw_messages = self.client.messages(self.channel, from_date, offset) + messages, total = self.parse_messages(raw_messages) + + for message in messages: + message["channel_info"] = channel_info + nmsgs += 1 + yield message + + offset += len(messages) + + if offset == total: + fetching = False + + logger.info("Fetch process completed: %s message fetched", nmsgs) + + @staticmethod + def parse_messages(raw_messages): + """Parse a channel messages JSON stream. + + This method parses a JSON stream, containing the + history of a channel. It returns a list of messages + and the total messages count in that channel. + + :param raw_messages: JSON string to parse + + :returns: a tuple with a list of dicts with the parsed messages + and a total messages count in the channel. + """ + result = json.loads(raw_messages) + return result['messages'], result['total'] + + @staticmethod + def parse_channel_info(raw_channel_info): + """Parse a channel's information JSON stream. + + This method parses a JSON stream, containing the information + of the channel, and returns a dict with the parsed data. + + :param raw_channel_info: JSON string to parse + + :returns: a dict with the parsed channel's information + """ + result = json.loads(raw_channel_info) + return result['channel'] + + @classmethod + def has_archiving(cls): + """Returns whether it supports archiving items on the fetch process. + + :returns: this backend supports items archive + """ + return True + + @classmethod + def has_resuming(cls): + """Returns whether it supports to resume the fetch process. + + :returns: this backend supports items resuming + """ + return True + + @staticmethod + def metadata_id(item): + """Extracts the identifier from a Rocket.Chat item.""" + + return item["_id"] + + @staticmethod + def metadata_updated_on(item): + """Extracts the update time from a Rocket.Chat item. + + The timestamp is extracted from 'ts' field, + and then converted into a UNIX timestamp. + + :param item: item generated by the backend + + :returns: extracted timestamp + """ + ts = str_to_datetime(item['_updatedAt']).timestamp() + return ts + + @staticmethod + def metadata_category(item): + """Extracts the category from a Rocket.Chat item. + + This backend only generates one type of item which is + 'message'. + """ + return CATEGORY_MESSAGE + + def _init_client(self, from_archive=False): + """Init client""" + + return RocketChatClient(self.url, self.user_id, self.api_token, + self.max_items, self.sleep_for_rate, + self.min_rate_to_sleep, from_archive, self.archive, self.ssl_verify) + + +class RocketChatClient(HttpClient, RateLimitHandler): + """Rocket.Chat API client. + + Client for fetching information from the Rocket.Chat server + using its REST API. + + :param url: server url from where messages are to be fetched + :param user_id: generated User Id using your Rocket.Chat account + :param api_token: token needed to use the API + :param max_items: maximum number of message requested on the same query + :param sleep_for_rate: sleep until rate limit is reset + :param min_rate_to_sleep: minimum rate needed to sleep until + it will be reset + :param from_archive: it tells whether to write/read the archive + :param archive: archive to store/retrieve items + :param ssl_verify: enable/disable SSL verification + """ + RCHANNEL_MESSAGES = 'channels.messages' + RCHANNEL_INFO = 'channels.info' + + HAUTH_TOKEN = 'X-Auth-Token' + HUSER_ID = 'X-User-Id' + + PCHANNEL_NAME = 'roomName' + PCOUNT = "count" + POLDEST = "oldest" + + def __init__(self, url, user_id, api_token, max_items=MAX_ITEMS, + sleep_for_rate=False, min_rate_to_sleep=MIN_RATE_LIMIT, + from_archive=False, archive=None, ssl_verify=True): + + base_url = urijoin(url, API_EXTENSION) + self.user_id = user_id + self.api_token = api_token + self.max_items = max_items + + super().__init__(base_url, archive=archive, from_archive=from_archive, + ssl_verify=ssl_verify) + super().setup_rate_limit_handler(sleep_for_rate=sleep_for_rate, min_rate_to_sleep=min_rate_to_sleep) + + def calculate_time_to_reset(self): + """Number of seconds to wait. They are contained in the rate limit reset header.""" + + time_to_reset = self.rate_limit_reset_ts - (datetime_utcnow().replace(microsecond=0).timestamp() + 1) * 1000 + time_to_reset /= 1000 + + if time_to_reset < 0: + time_to_reset = 0 + + return time_to_reset + + def channel_info(self, channel): + """Fetch information about a channel.""" + + params = { + self.PCHANNEL_NAME: channel, + } + + path = urijoin(self.base_url, self.RCHANNEL_INFO) + response = self.fetch(path, params) + + return response + + def messages(self, channel, from_date, offset): + """Fetch messages from a channel. + + The messages are fetch in ascending order i.e. from the oldest + to the latest based on the time they were last updated. A query is + also passed as a param to fetch the messages from a given date. + """ + query = '{"_updatedAt": {"$gte": {"$date": "%s"}}}' % from_date.isoformat() + + # The 'sort' param accepts a field based on which the messages are sorted. + # The value of the field can be 1 for ascending order or -1 for descending order. + params = { + "roomName": channel, + "sort": '{"_updatedAt": 1}', + "count": self.max_items, + "offset": offset, + "query": query + } + + path = urijoin(self.base_url, self.RCHANNEL_MESSAGES) + response = self.fetch(path, params) + + return response + + def fetch(self, url, payload=None, headers=None): + """Fetch the data from a given URL. + + :param url: link to the resource + :param payload: payload of the request + :param headers: headers of the request + + :returns a response object + """ + headers = { + self.HAUTH_TOKEN: self.api_token, + self.HUSER_ID: self.user_id + } + + logger.debug("Rocket.Chat client message request with params: %s", str(payload)) + + if not self.from_archive: + self.sleep_for_rate_limit() + + response = super().fetch(url, payload, headers=headers) + + if not self.from_archive: + self.update_rate_limit(response) + + return response.text + + @staticmethod + def sanitize_for_archive(url, headers, payload): + """Sanitize payload of a HTTP request by removing the token and + user id information before storing/retrieving archived items. + + :param: url: HTTP url request + :param: headers: HTTP headers request + :param: payload: HTTP payload request + + :returns: url, headers and the sanitized payload + """ + if RocketChatClient.HAUTH_TOKEN in headers: + headers.pop(RocketChatClient.HAUTH_TOKEN) + + if RocketChatClient.HUSER_ID in headers: + headers.pop(RocketChatClient.HUSER_ID) + + return url, headers, payload + + +class RocketChatCommand(BackendCommand): + """Class to run Rocket.Chat backend from the command line.""" + + BACKEND = RocketChat + + @classmethod + def setup_cmd_parser(cls): + """Returns the Rocket.Chat argument parser.""" + + parser = BackendCommandArgumentParser(cls.BACKEND, + from_date=True, + token_auth=True, + archive=True, + ssl_verify=True) + + # Backend token is required + action = parser.parser._option_string_actions['--api-token'] + action.required = True + + parser.parser.add_argument('-u', '--user-id', dest='user_id', + required=True, + help="User Id to fetch messages") + + # Required positional arguments + parser.parser.add_argument('url', + help="URL of the Rocket.Chat server") + + parser.parser.add_argument('channel', + help="Rocket.Chat channel(room) name") + + # Rocket.Chat options + group = parser.parser.add_argument_group('Rocket.Chat arguments') + group.add_argument('--max-items', dest='max_items', + type=int, default=MAX_ITEMS, + help="Maximum number of items requested on the same query") + group.add_argument('--sleep-for-rate', dest='sleep_for_rate', + action='store_true', + help="sleep for getting more rate") + group.add_argument('--min-rate-to-sleep', dest='min_rate_to_sleep', + default=MIN_RATE_LIMIT, type=int, + help="sleep until reset when the rate limit reaches this value") + + return parser diff --git a/releases/unreleased/add-rocket.chat-backend.yml b/releases/unreleased/add-rocket.chat-backend.yml new file mode 100644 index 000000000..fe6a68aed --- /dev/null +++ b/releases/unreleased/add-rocket.chat-backend.yml @@ -0,0 +1,13 @@ +--- +title: Add Rocket.Chat backend +category: added +author: Animesh Kumar +issue: 543 +notes: > + Added support to fetch messages from a Rocket.Chat + channel. The messages are fetched in an ascending + order based on time when they were last updated. + The channel information is also fetched. + + The tests have been added accordingly. + The usage docs have been updated. diff --git a/tests/data/rocketchat/channel_info.json b/tests/data/rocketchat/channel_info.json new file mode 100644 index 000000000..ad44bcccc --- /dev/null +++ b/tests/data/rocketchat/channel_info.json @@ -0,0 +1,38 @@ +{ + "channel": { + "_id": "wyJHNAtuPGnQCT5xP", + "name": "testapichannel", + "fname": "testapichannel", + "t": "c", + "msgs": 3, + "usersCount": 2, + "u": { + "_id": "123user", + "username": "animesh_username1" + }, + "customFields": {}, + "broadcast": false, + "encrypted": false, + "ts": "2020-05-03T07:30:30.990Z", + "ro": false, + "default": false, + "sysMes": true, + "_updatedAt": "2020-05-03T07:32:03.594Z", + "lastMessage": { + "_id": "p5dQSb48W25EimhJK", + "rid": "wyJHNAtuPGnQCT5xP", + "msg": "Test message 2", + "ts": "2020-05-03T07:32:03.571Z", + "u": { + "_id": "123user", + "username": "animeshk_username1", + "name": "Animesh Kumar" + }, + "_updatedAt": "2020-05-03T07:32:03.587Z", + "mentions": [], + "channels": [] + }, + "lm": "2020-05-03T07:32:03.571Z" + }, + "success": true +} \ No newline at end of file diff --git a/tests/data/rocketchat/message_empty_2020_05_10.json b/tests/data/rocketchat/message_empty_2020_05_10.json new file mode 100644 index 000000000..cf8a847e4 --- /dev/null +++ b/tests/data/rocketchat/message_empty_2020_05_10.json @@ -0,0 +1,7 @@ +{ + "messages": [], + "count": 0, + "offset": 0, + "total": 0, + "success": true +} \ No newline at end of file diff --git a/tests/data/rocketchat/message_page_1.json b/tests/data/rocketchat/message_page_1.json new file mode 100644 index 000000000..206a005cd --- /dev/null +++ b/tests/data/rocketchat/message_page_1.json @@ -0,0 +1,62 @@ +{ + "messages": [ + { + "_id": "4AwA2eJQ7xBgPZ4mv", + "rid": "wyJHNAtuPGnQCT5xP", + "msg": "Test message 1", + "ts": "2020-05-02T07:30:42.993Z", + "u": { + "_id": "123user", + "username": "animesh_username1", + "name": "Animesh Kumar" + }, + "_updatedAt": "2020-05-02T07:31:26.164Z", + "mentions": [], + "channels": [], + "reactions": { + ":cool:": { + "usernames": [ + "animesh_username2" + ] + }, + ":wait:": { + "usernames": [ + "animesh_username2" + ] + } + }, + "starred": [], + "replies": [ + "567user" + ], + "tcount": 1, + "tlm": "2020-05-02T07:31:26.007Z" + }, + { + "_id": "zofFonHMq5M3tdGyu", + "rid": "wyJHNAtuPGnQCT5xP", + "tmid": "4AwA2eJQ7xBgPZ4mv", + "msg": "Test reply 1", + "ts": "2020-05-02T07:31:26.007Z", + "u": { + "_id": "567user", + "username": "animesh_username2", + "name": "Animesh Kumar Singh" + }, + "_updatedAt": "2020-05-02T07:31:56.711Z", + "mentions": [], + "channels": [], + "_hidden": true, + "parent": "WnWSwiD877xRpqcMb", + "editedAt": "2020-05-02T07:31:56.711Z", + "editedBy": { + "_id": "123user", + "username": "animesh_username1" + } + } + ], + "count": 2, + "offset": 0, + "total": 4, + "success": true +} \ No newline at end of file diff --git a/tests/data/rocketchat/message_page_2.json b/tests/data/rocketchat/message_page_2.json new file mode 100644 index 000000000..327e4daec --- /dev/null +++ b/tests/data/rocketchat/message_page_2.json @@ -0,0 +1,43 @@ +{ + "messages": [ + { + "_id": "WnWSwiD877xRpqcMb", + "rid": "wyJHNAtuPGnQCT5xP", + "tmid": "4AwA2eJQ7xBgPZ4mv", + "msg": "Test reply 1 edited", + "ts": "2020-05-02T07:31:26.007Z", + "u": { + "_id": "567user", + "username": "animesh_username2", + "name": "Animesh Kumar Singh" + }, + "_updatedAt": "2020-05-02T07:31:56.714Z", + "mentions": [], + "channels": [], + "editedAt": "2020-05-02T07:31:56.712Z", + "editedBy": { + "_id": "123user", + "username": "animesh_username1" + }, + "urls": [] + }, + { + "_id": "p5dQSb48W25EimhJK", + "rid": "wyJHNAtuPGnQCT5xP", + "msg": "Test message 2", + "ts": "2020-05-03T07:32:03.571Z", + "u": { + "_id": "123user", + "username": "animesh_username1", + "name": "Animesh Kumar" + }, + "_updatedAt": "2020-05-03T07:32:03.587Z", + "mentions": [], + "channels": [] + } + ], + "count": 2, + "offset": 2, + "total": 4, + "success": true +} \ No newline at end of file diff --git a/tests/data/rocketchat/message_page_2020_05_03.json b/tests/data/rocketchat/message_page_2020_05_03.json new file mode 100644 index 000000000..f05dc3d04 --- /dev/null +++ b/tests/data/rocketchat/message_page_2020_05_03.json @@ -0,0 +1,22 @@ +{ + "messages": [ + { + "_id": "p5dQSb48W25EimhJK", + "rid": "wyJHNAtuPGnQCT5xP", + "msg": "Test message 2", + "ts": "2020-05-03T07:32:03.571Z", + "u": { + "_id": "123user", + "username": "animesh_username1", + "name": "Animesh Kumar" + }, + "_updatedAt": "2020-05-03T07:32:03.587Z", + "mentions": [], + "channels": [] + } + ], + "count": 1, + "offset": 0, + "total": 1, + "success": true +} \ No newline at end of file diff --git a/tests/test_rocketchat.py b/tests/test_rocketchat.py new file mode 100644 index 000000000..7ac6f4094 --- /dev/null +++ b/tests/test_rocketchat.py @@ -0,0 +1,564 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (C) 2015-2020 Bitergia +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Authors: +# Aditya Prajapati +# Animesh Kumar +# + +import copy +import datetime +import httpretty +import os +import pkg_resources +import unittest +import unittest.mock +import dateutil.tz +import time +import json + +pkg_resources.declare_namespace('perceval.backends') + +from perceval.backend import BackendCommandArgumentParser +from perceval.errors import RateLimitError +from perceval.utils import DEFAULT_DATETIME +from perceval.backends.core.rocketchat import (RocketChat, + RocketChatClient, + RocketChatCommand, + MIN_RATE_LIMIT, + MAX_ITEMS) + +from base import TestCaseBackendArchive + +ROCKETCHAT_SERVER_URL = 'https://open.rocket.chat' +ROCKETCHAT_CHANNEL_NAME = 'testapichannel' +ROCKETCHAT_API_EXTENSION = "/api/v1/" +ROCKETCHAT_API_BASE_URL = ROCKETCHAT_SERVER_URL + ROCKETCHAT_API_EXTENSION +ROCKETCHAT_MESSAGE_URL = ROCKETCHAT_API_BASE_URL + RocketChatClient.RCHANNEL_MESSAGES +ROCKETCHAT_CHANNEL_URL = ROCKETCHAT_API_BASE_URL + RocketChatClient.RCHANNEL_INFO + + +def setup_http_server(no_message=False, rate_limit_headers=None, from_date=False): + """Setup a mock HTTP server""" + + message_page_1 = read_file('data/rocketchat/message_page_1.json') + message_page_2 = read_file('data/rocketchat/message_page_2.json') + channel_info = read_file('data/rocketchat/channel_info.json') + message_empty_2020_5_10 = read_file('data/rocketchat/message_empty_2020_05_10.json') + message_page_2020_05_03 = read_file('data/rocketchat/message_page_2020_05_03.json') + + if not rate_limit_headers: + rate_limit_headers = {} + + httpretty.register_uri(httpretty.GET, + ROCKETCHAT_CHANNEL_URL + '?roomName=testapichannel', + body=channel_info, + status=200, + forcing_headers=rate_limit_headers) + + roomName = '?roomName=testapichannel' + sort = '&sort={"_updatedAt": 1}' + count = '&count=2' + params = roomName + sort + count + + if no_message: + query = '&q={"_updatedAt": {"$gte": {"$date": "2020-05-10T00:00:00+00:00"}}}' + httpretty.register_uri(httpretty.GET, + ROCKETCHAT_MESSAGE_URL + params + query + '&offset=0', + body=message_empty_2020_5_10, + status=200, + forcing_headers=rate_limit_headers) + elif from_date: + query = '&q={"_updatedAt": {"$gte": {"$date": "2020-05-03T00:00:00+00:00"}}}' + httpretty.register_uri(httpretty.GET, + ROCKETCHAT_MESSAGE_URL + params + query + '&offset=0', + body=message_page_2020_05_03, + status=200, + forcing_headers=rate_limit_headers) + else: + query = '&q={"_updatedAt": {"$gte": {"$date": "2020-05-02T00:00:00+00:00"}}}' + httpretty.register_uri(httpretty.GET, + ROCKETCHAT_MESSAGE_URL + params + query + '&offset=2', + body=message_page_2, + status=200, + forcing_headers=rate_limit_headers) + + httpretty.register_uri(httpretty.GET, + ROCKETCHAT_MESSAGE_URL + params + query + '&offset=0', + body=message_page_1, + status=200, + forcing_headers=rate_limit_headers) + + +class MockedRocketChatClient(RocketChatClient): + """Mocked Rocket.Chat client for testing""" + + def __init__(self, url, user_id, api_token, max_items=MAX_ITEMS, archive=None, + sleep_for_rate=False, min_rate_to_sleep=MIN_RATE_LIMIT, + from_archive=False, ssl_verify=True): + super().__init__(url, user_id, api_token, max_items=max_items, + min_rate_to_sleep=min_rate_to_sleep, + sleep_for_rate=sleep_for_rate, + archive=archive, + from_archive=from_archive, + ssl_verify=ssl_verify + ) + self.rate_limit_reset_ts = -1 + + +def read_file(filename): + with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), filename), 'rb') as f: + content = f.read() + return content + + +class TestRocketChatBackend(unittest.TestCase): + """Tests for Rocket.Chat backend class""" + + def test_initialization(self): + """Test whether attributes are initialized""" + + backend = RocketChat(url='https://chat.example.com', user_id='123user', api_token='aaa', + channel='testapichannel', tag='test') + + self.assertEqual(backend.url, 'https://chat.example.com') + self.assertEqual(backend.user_id, '123user') + self.assertEqual(backend.api_token, "aaa") + self.assertEqual(backend.channel, "testapichannel") + self.assertEqual(backend.min_rate_to_sleep, MIN_RATE_LIMIT) + self.assertIsNone(backend.client) + self.assertFalse(backend.sleep_for_rate) + self.assertTrue(backend.ssl_verify) + self.assertEqual(backend.origin, 'https://chat.example.com/testapichannel') + self.assertEqual(backend.tag, 'test') + self.assertEqual(backend.max_items, MAX_ITEMS) + + # When tag is empty or None it will be set to + # the value in URL + backend = RocketChat(url="https://chat.example.com", user_id="123user", api_token='aaa', + channel='testapichannel', tag=None) + self.assertEqual(backend.origin, 'https://chat.example.com/testapichannel') + self.assertEqual(backend.tag, 'https://chat.example.com/testapichannel') + + backend = RocketChat(url="https://chat.example.com", user_id="123user", api_token='aaa', + channel='testapichannel', tag='') + self.assertEqual(backend.origin, 'https://chat.example.com/testapichannel') + self.assertEqual(backend.tag, 'https://chat.example.com/testapichannel') + + backend = RocketChat(url='https://chat.example.com', user_id='123user', api_token='aaa', + channel='testapichannel', tag='', sleep_for_rate=True, + ssl_verify=False, max_items=20, min_rate_to_sleep=1) + self.assertEqual(backend.origin, 'https://chat.example.com/testapichannel') + self.assertEqual(backend.tag, 'https://chat.example.com/testapichannel') + self.assertFalse(backend.ssl_verify) + self.assertTrue(backend.sleep_for_rate) + self.assertEqual(backend.max_items, 20) + self.assertEqual(backend.min_rate_to_sleep, 1) + + def test_has_archiving(self): + """Test if it returns True when has_archiving is called""" + + self.assertEqual(RocketChat.has_archiving(), True) + + def test_has_resuming(self): + """Test if it returns True when has_resuming is called""" + + self.assertEqual(RocketChat.has_resuming(), True) + + @httpretty.activate + def test_fetch_messages(self): + """Test whether a list of messages is returned""" + + setup_http_server() + + backend = RocketChat(url='https://open.rocket.chat', user_id='123user', + api_token='aaa', channel='testapichannel') + messages = [m for m in backend.fetch()] + + self.assertEqual(len(messages), 4) + + message = messages[0] + self.assertEqual(message['data']['_id'], '4AwA2eJQ7xBgPZ4mv') + self.assertEqual(message['origin'], 'https://open.rocket.chat/testapichannel') + self.assertEqual(message['uuid'], '888b6c9a728c267435cee1d5fe8f5dbe446614a2') + self.assertEqual(message['updated_on'], 1588404686.164) + self.assertEqual(message['category'], 'message') + self.assertEqual(message['tag'], 'https://open.rocket.chat/testapichannel') + self.assertEqual(message['data']['msg'], 'Test message 1') + self.assertEqual(message['data']['u']['username'], 'animesh_username1') + self.assertEqual(message['data']['u']['name'], 'Animesh Kumar') + self.assertListEqual(message['data']['replies'], ["567user"]) + self.assertEqual(message['data']['channel_info']['_id'], 'wyJHNAtuPGnQCT5xP') + self.assertEqual(message['data']['channel_info']['lastMessage']['msg'], 'Test message 2') + self.assertEqual(message['data']['channel_info']['usersCount'], 2) + + message = messages[1] + self.assertEqual(message['data']['_id'], 'zofFonHMq5M3tdGyu') + self.assertEqual(message['origin'], 'https://open.rocket.chat/testapichannel') + self.assertEqual(message['uuid'], '6fa3f6a18491b9bcb3309f12e3a2b1cd654980c3') + self.assertEqual(message['updated_on'], 1588404716.711) + self.assertEqual(message['category'], 'message') + self.assertEqual(message['tag'], 'https://open.rocket.chat/testapichannel') + self.assertEqual(message['data']['msg'], 'Test reply 1') + self.assertEqual(message['data']['u']['username'], 'animesh_username2') + self.assertEqual(message['data']['u']['name'], 'Animesh Kumar Singh') + self.assertEqual(message['data']['channel_info']['_id'], 'wyJHNAtuPGnQCT5xP') + self.assertEqual(message['data']['channel_info']['lastMessage']['msg'], 'Test message 2') + self.assertEqual(message['data']['channel_info']['usersCount'], 2) + + message = messages[2] + self.assertEqual(message['data']['_id'], 'WnWSwiD877xRpqcMb') + self.assertEqual(message['origin'], 'https://open.rocket.chat/testapichannel') + self.assertEqual(message['uuid'], '3b3afa62a63766ebeb70e3c8951c2c4b42f34767') + self.assertEqual(message['updated_on'], 1588404716.714) + self.assertEqual(message['category'], 'message') + self.assertEqual(message['tag'], 'https://open.rocket.chat/testapichannel') + self.assertEqual(message['data']['msg'], 'Test reply 1 edited') + self.assertEqual(message['data']['u']['username'], 'animesh_username2') + self.assertEqual(message['data']['u']['name'], 'Animesh Kumar Singh') + self.assertEqual(message['data']['editedBy']['username'], 'animesh_username1') + self.assertEqual(message['data']['channel_info']['_id'], 'wyJHNAtuPGnQCT5xP') + self.assertEqual(message['data']['channel_info']['usersCount'], 2) + + message = messages[3] + self.assertEqual(message['data']['_id'], 'p5dQSb48W25EimhJK') + self.assertEqual(message['origin'], 'https://open.rocket.chat/testapichannel') + self.assertEqual(message['uuid'], 'a3b2a4c195e8b6a155cf5eddb2b8f79a13f836dd') + self.assertEqual(message['updated_on'], 1588491123.587) + self.assertEqual(message['category'], 'message') + self.assertEqual(message['tag'], 'https://open.rocket.chat/testapichannel') + self.assertEqual(message['data']['msg'], 'Test message 2') + self.assertEqual(message['data']['u']['username'], 'animesh_username1') + self.assertEqual(message['data']['u']['name'], 'Animesh Kumar') + self.assertEqual(message['data']['channel_info']['_id'], 'wyJHNAtuPGnQCT5xP') + self.assertEqual(message['data']['channel_info']['usersCount'], 2) + + @httpretty.activate + def test_fetch_from_date(self): + """Test when fetching messages from a given date""" + + setup_http_server(from_date=True) + from_date = datetime.datetime(2020, 5, 3, 0, 0, tzinfo=dateutil.tz.tzutc()) + + backend = RocketChat(url='https://open.rocket.chat', user_id='123user', + api_token='aaa', channel='testapichannel') + + messages = [m for m in backend.fetch(from_date=from_date)] + self.assertEqual(len(messages), 1) + + message = messages[0] + self.assertEqual(message['data']['_id'], 'p5dQSb48W25EimhJK') + self.assertEqual(message['origin'], 'https://open.rocket.chat/testapichannel') + self.assertEqual(message['uuid'], 'a3b2a4c195e8b6a155cf5eddb2b8f79a13f836dd') + self.assertEqual(message['updated_on'], 1588491123.587) + self.assertEqual(message['category'], 'message') + self.assertEqual(message['tag'], 'https://open.rocket.chat/testapichannel') + self.assertEqual(message['data']['msg'], 'Test message 2') + self.assertEqual(message['data']['u']['username'], 'animesh_username1') + self.assertEqual(message['data']['u']['name'], 'Animesh Kumar') + self.assertEqual(message['data']['channel_info']['_id'], 'wyJHNAtuPGnQCT5xP') + self.assertEqual(message['data']['channel_info']['usersCount'], 2) + + @httpretty.activate + def test_search_fields_messages(self): + """Test whether the search_fields is properly set""" + + setup_http_server() + + backend = RocketChat(url='https://open.rocket.chat', user_id='123user', + api_token='aaa', channel='testapichannel') + messages = [m for m in backend.fetch()] + + message = messages[0] + self.assertEqual(message['search_fields']['item_id'], backend.metadata_id(message['data'])) + self.assertEqual(message['search_fields']['channel_id'], 'wyJHNAtuPGnQCT5xP') + self.assertEqual(message['search_fields']['channel_name'], 'testapichannel') + + @httpretty.activate + def test_fetch_empty(self): + """Test whether an empty list is returned when there are no messages""" + + setup_http_server(no_message=True) + + backend = RocketChat(url='https://open.rocket.chat', user_id='123user', + api_token='aaa', channel='testapichannel') + messages = [m for m in backend.fetch()] + self.assertListEqual(messages, []) + + +class TestRocketChatBackendArchive(TestCaseBackendArchive): + """Rocket.Chat backend tests using an archive""" + + def setUp(self): + super().setUp() + self.backend_write_archive = RocketChat(url='https://open.rocket.chat', user_id='123user', + api_token='aaa', channel='testapichannel', + max_items=5, archive=self.archive) + self.backend_read_archive = RocketChat(url='https://open.rocket.chat', user_id='123user', + api_token='aaa', channel='testapichannel', + max_items=5, archive=self.archive) + + @httpretty.activate + @unittest.mock.patch('perceval.backends.core.rocketchat.datetime_utcnow') + def test_fetch_from_archive(self, mock_utcnow): + """Test if a list of messages is returned from archive""" + + mock_utcnow.return_value = datetime.datetime(2020, 1, 1, + tzinfo=dateutil.tz.tzutc()) + + setup_http_server() + self._test_fetch_from_archive(from_date=None) + + @httpretty.activate + @unittest.mock.patch('perceval.backends.core.rocketchat.datetime_utcnow') + def test_fetch_from_date_from_archive(self, mock_utcnow): + """Test whether a list of messages is returned from archive after a given date""" + + mock_utcnow.return_value = datetime.datetime(2020, 1, 1, + tzinfo=dateutil.tz.tzutc()) + + setup_http_server() + + from_date = datetime.datetime(2020, 5, 3, 18, 35, 40, 69, + tzinfo=dateutil.tz.tzutc()) + self._test_fetch_from_archive(from_date=from_date) + + @httpretty.activate + @unittest.mock.patch('perceval.backends.core.rocketchat.datetime_utcnow') + def test_fetch_empty_from_archive(self, mock_utcnow): + """Test whether no messages are returned when the archive is empty""" + + mock_utcnow.return_value = datetime.datetime(2020, 1, 1, + tzinfo=dateutil.tz.tzutc()) + + setup_http_server() + + from_date = datetime.datetime(2020, 5, 3, + tzinfo=dateutil.tz.tzutc()) + self._test_fetch_from_archive(from_date=from_date) + + +class TestRocketChatClient(unittest.TestCase): + """Tests for RocketChatClient class""" + + def test_init(self): + """Check attributes initialization""" + + client = RocketChatClient(url='https://open.rocket.chat', user_id='123user', + api_token='aaa', ssl_verify=True) + self.assertIsInstance(client, RocketChatClient) + self.assertEqual(client.base_url, 'https://open.rocket.chat/api/v1') + self.assertEqual(client.user_id, '123user') + self.assertEqual(client.api_token, 'aaa') + self.assertTrue(client.ssl_verify) + self.assertFalse(client.sleep_for_rate) + self.assertEqual(client.min_rate_to_sleep, MIN_RATE_LIMIT) + + client = RocketChatClient(url='https://open.rocket.chat', user_id='123user', api_token='aaa', + sleep_for_rate=True, min_rate_to_sleep=1, ssl_verify=False) + self.assertIsInstance(client, RocketChatClient) + self.assertEqual(client.base_url, 'https://open.rocket.chat/api/v1') + self.assertEqual(client.user_id, '123user') + self.assertEqual(client.api_token, 'aaa') + self.assertFalse(client.ssl_verify) + self.assertTrue(client.sleep_for_rate) + self.assertEqual(client.min_rate_to_sleep, 1) + + @httpretty.activate + def test_messages(self): + """Test whether messages are fetched""" + + setup_http_server() + + client = RocketChatClient(url='https://open.rocket.chat', user_id='123user', + api_token='aaa', ssl_verify=True) + + messages = client.messages('testapichannel', from_date=DEFAULT_DATETIME, offset=0) + messages = json.loads(messages) + + self.assertEqual(len(messages['messages']), 2) + + # Check requests + expected = { + 'count': ['100'], + 'offset': ['0'], + 'query': ['{"_updatedAt": {"$gte": {"$date": "1970-01-01T00:00:00 00:00"}}}'], + 'roomName': ['testapichannel'], + 'sort': ['{"_updatedAt": 1}'] + } + + self.assertEqual(httpretty.last_request().querystring, expected) + self.assertEqual(httpretty.last_request().headers[RocketChatClient.HAUTH_TOKEN], 'aaa') + + @httpretty.activate + def test_channel_info(self): + """Test whether channel information is fetched""" + + setup_http_server() + + client = RocketChatClient(url='https://open.rocket.chat', user_id='123user', + api_token='aaa', ssl_verify=True) + channel = client.channel_info('testapichannel') + channel = json.loads(channel) + + self.assertEqual(channel['channel']['_id'], 'wyJHNAtuPGnQCT5xP') + self.assertEqual(channel['channel']['name'], 'testapichannel') + self.assertEqual(channel['channel']['usersCount'], 2) + self.assertEqual(channel['channel']['msgs'], 3) + self.assertEqual(channel['channel']['lastMessage']['msg'], 'Test message 2') + + # Check requests + expected = { + 'roomName': ['testapichannel'], + } + + self.assertEqual(httpretty.last_request().querystring, expected) + self.assertEqual(httpretty.last_request().headers[RocketChatClient.HAUTH_TOKEN], 'aaa') + + def test_calculate_time_to_reset(self): + """Test whether the time to reset is zero if the sleep time is negative""" + + client = MockedRocketChatClient(url='https://open.rocket.chat', user_id='123user', + api_token='aaa', max_items=10, archive=None, from_archive=False, + min_rate_to_sleep=2, sleep_for_rate=True, ssl_verify=False) + time_to_reset = client.calculate_time_to_reset() + self.assertEqual(time_to_reset, 0) + + @httpretty.activate + def test_sleep_for_rate(self): + """Test if the clients sleeps when the rate limit is reached""" + + wait = 10000 + reset = int(time.time() * 1000 + wait) + rate_limit_headers = {'X-RateLimit-Remaining': '0', + 'X-RateLimit-Reset': reset} + + setup_http_server(rate_limit_headers=rate_limit_headers) + + client = RocketChatClient(url='https://open.rocket.chat', user_id='123user', + api_token='aaa', min_rate_to_sleep=5, + sleep_for_rate=True) + + _ = client.channel_info('testapichannel') + after = float(time.time() * 1000) + + self.assertTrue(reset >= after) + + @httpretty.activate + def test_rate_limit_error(self): + """Test if a rate limit error is raised when rate is exhausted""" + + wait = 2000 + reset = int(time.time() * 1000 + wait) + rate_limit_headers = {'X-RateLimit-Remaining': '0', + 'X-RateLimit-Reset': reset} + + setup_http_server(rate_limit_headers=rate_limit_headers) + + client = RocketChatClient(url='https://open.rocket.chat', user_id='123user', + api_token='aaa', sleep_for_rate=False) + + _ = client.channel_info('testapichannel') + with self.assertRaises(RateLimitError): + _ = client.messages('testapichannel', from_date=DEFAULT_DATETIME, offset=0) + + def test_sanitize_for_archive(self): + """Test whether the sanitize method works properly""" + + url = "https://open.rocket.chat/testapichannel" + headers = { + RocketChatClient.HAUTH_TOKEN: 'aaaa', + RocketChatClient.HUSER_ID: '123user' + } + + payload = { + 'count': 100, + 'offset': 0, + 'query': '{"_updatedAt": {"$gte": {"$date": "1970-01-01T00:00:00 00:00"}}}', + 'roomName': 'testapichannel', + 'sort': '{"_updatedAt": 1}' + } + + s_url, s_headers, s_payload = RocketChatClient.sanitize_for_archive(url, copy.deepcopy(headers), payload) + headers.pop(RocketChatClient.HAUTH_TOKEN) + headers.pop(RocketChatClient.HUSER_ID) + + self.assertEqual(url, s_url) + self.assertEqual(headers, s_headers) + self.assertEqual(payload, s_payload) + + +class TestRocketChatCommand(unittest.TestCase): + """Tests for RocketChatCommand class""" + + def test_backend_class(self): + """Test if the backend class is RocketChat""" + + self.assertIs(RocketChatCommand.BACKEND, RocketChat) + + def test_setup_cmd_parser(self): + """Test if it parser object is correctly initialized""" + + parser = RocketChatCommand.setup_cmd_parser() + self.assertIsInstance(parser, BackendCommandArgumentParser) + self.assertEqual(parser._backend, RocketChat) + + args = ['-t', 'aaa', + '-u', '123user', + '--tag', 'test', + '--sleep-for-rate', + '--from-date', '1970-01-01', + 'https://open.rocket.chat', + 'testapichannel'] + + parsed_args = parser.parse(*args) + self.assertEqual(parsed_args.api_token, 'aaa') + self.assertEqual(parsed_args.user_id, '123user') + self.assertEqual(parsed_args.url, 'https://open.rocket.chat') + self.assertEqual(parsed_args.channel, 'testapichannel') + self.assertEqual(parsed_args.tag, 'test') + self.assertEqual(parsed_args.from_date, DEFAULT_DATETIME) + self.assertTrue(parsed_args.ssl_verify) + self.assertTrue(parsed_args.sleep_for_rate) + + from_date = datetime.datetime(2020, 3, 1, 0, 0, tzinfo=dateutil.tz.tzutc()) + + args = ['-t', 'aaa', + '-u', '123user', + '--tag', 'test', + '--max-items', '10', + '--no-ssl-verify', + '--min-rate-to-sleep', '1', + '--from-date', '2020-03-01', + 'https://open.rocket.chat', + 'testapichannel'] + + parsed_args = parser.parse(*args) + self.assertEqual(parsed_args.api_token, 'aaa') + self.assertEqual(parsed_args.user_id, '123user') + self.assertEqual(parsed_args.url, 'https://open.rocket.chat') + self.assertEqual(parsed_args.channel, 'testapichannel') + self.assertEqual(parsed_args.tag, 'test') + self.assertEqual(parsed_args.from_date, from_date) + self.assertEqual(parsed_args.min_rate_to_sleep, 1) + self.assertFalse(parsed_args.ssl_verify) + self.assertFalse(parsed_args.sleep_for_rate) + + +if __name__ == "__main__": + unittest.main(warnings='ignore')