diff --git a/README.md b/README.md
index e905ef302..23b0057d0 100644
--- a/README.md
+++ b/README.md
@@ -36,6 +36,7 @@ are:
phabricator Fetch tasks from a Phabricator site
pipermail Fetch messages from a Pipermail archiver
redmine Fetch issues from a Redmine server
+ rocketchat Fetch messages from a Rocket.Chat channel
rss Fetch entries from a RSS feed server
slack Fetch messages from a Slack channel
stackexchange Fetch questions from StackExchange sites
@@ -321,6 +322,13 @@ $ perceval pipermail 'http://mail-archives.apache.org/mod_mbox/httpd-dev/'
$ perceval redmine 'https://www.redmine.org/' --from-date '2016-01-01' -t abcdefghijk
```
+### Rocket.Chat
+
+Rocket.Chat backend needs an API token and a User Id to authenticate to the server.
+```
+$ perceval rocketchat -t 'abchdefghij' -u '1234abcd' --from-date '2020-05-02' https://open.rocket.chat general
+```
+
### RSS
```
$ perceval rss 'https://blog.bitergia.com/feed/'
diff --git a/bin/perceval b/bin/perceval
index bf223ddf1..98e9344ba 100755
--- a/bin/perceval
+++ b/bin/perceval
@@ -73,6 +73,7 @@ are:
phabricator Fetch tasks from a Phabricator site
pipermail Fetch messages from a Pipermail archiver
redmine Fetch issues from a Redmine server
+ rocketchat Fetch messages from a Rocket.Chat channel
rss Fetch entries from a RSS feed server
slack Fetch messages from a Slack channel
stackexchange Fetch questions from StackExchange sites
diff --git a/perceval/backends/core/rocketchat.py b/perceval/backends/core/rocketchat.py
new file mode 100644
index 000000000..f1c890e3c
--- /dev/null
+++ b/perceval/backends/core/rocketchat.py
@@ -0,0 +1,404 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2015-2020 Bitergia
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Aditya Prajapati <aditya10699@gmail.com>
+# Animesh Kumar <animuz111@gmail.com>
+#
+
+import logging
+import json
+
+from grimoirelab_toolkit.uris import urijoin
+from grimoirelab_toolkit.datetime import (datetime_utcnow,
+ datetime_to_utc,
+ str_to_datetime)
+
+from ...backend import (Backend,
+ BackendCommand,
+ BackendCommandArgumentParser)
+from ...client import HttpClient, RateLimitHandler
+from ...utils import DEFAULT_DATETIME
+
+CATEGORY_MESSAGE = "message"
+
+API_EXTENSION = "/api/v1/"
+
+MIN_RATE_LIMIT = 10
+MAX_ITEMS = 100
+
+logger = logging.getLogger(__name__)
+
+
+class RocketChat(Backend):
+ """Rocket.Chat backend.
+
+ This class allows to fetch messages from a channel(room) on a Rocket.Chat server.
+ An API token and a User Id is required to access the server.
+
+ :param url: server url from where messages are to be fetched
+ :param channel: name of the channel from where data will be fetched
+ :param user_id: generated User Id using your Rocket.Chat account
+ :param api_token: token needed to use the API
+ :param max_items: maximum number of message requested on the same query
+ :param sleep_for_rate: sleep until rate limit is reset
+ :param min_rate_to_sleep: minimum rate needed to sleep until
+ it will be reset
+ :param tag: label used to mark the data
+ :param archive: archive to store/retrieve items
+ :param ssl_verify: enable/disable SSL verification
+ """
+ version = '0.1.0'
+
+ CATEGORIES = [CATEGORY_MESSAGE]
+ EXTRA_SEARCH_FIELDS = {
+ 'channel_name': ['channel_info', 'name'],
+ 'channel_id': ['channel_info', '_id']
+ }
+
+ def __init__(self, url, channel, user_id, api_token, max_items=MAX_ITEMS,
+ sleep_for_rate=False, min_rate_to_sleep=MIN_RATE_LIMIT,
+ tag=None, archive=None, ssl_verify=True):
+ origin = urijoin(url, channel)
+
+ super().__init__(origin, tag=tag, archive=archive, ssl_verify=ssl_verify)
+
+ self.url = url
+ self.channel = channel
+ self.user_id = user_id
+ self.api_token = api_token
+ self.max_items = max_items
+ self.sleep_for_rate = sleep_for_rate
+ self.min_rate_to_sleep = min_rate_to_sleep
+ self.client = None
+
+ def fetch(self, category=CATEGORY_MESSAGE, from_date=DEFAULT_DATETIME, filter_classified=False):
+ """Fetch the messages from the channel.
+
+ This method fetches the messages stored on the channel that were
+ sent since the given date.
+
+ :param category: the category of items to fetch
+ :param from_date: obtain messages sent since this date
+ :param filter_classified: remove classified fields from the resulting items
+
+ :returns: a generator of messages
+ """
+ if not from_date:
+ from_date = DEFAULT_DATETIME
+ from_date = datetime_to_utc(from_date)
+
+ kwargs = {'from_date': from_date}
+
+ items = super().fetch(category, **kwargs)
+
+ return items
+
+ def fetch_items(self, category, **kwargs):
+ """Fetch the messages.
+
+ :param category: the category of items to fetch
+ :param kwargs: backend arguments
+
+ :returns: a generator of items
+ """
+ from_date = kwargs['from_date']
+ logger.info("Fetching messages of channel: %s from date: %s",
+ self.channel, from_date)
+
+ raw_channel_info = self.client.channel_info(self.channel)
+ channel_info = self.parse_channel_info(raw_channel_info)
+
+ fetching = True
+ nmsgs = 0
+ offset = 0
+
+ while fetching:
+ raw_messages = self.client.messages(self.channel, from_date, offset)
+ messages, total = self.parse_messages(raw_messages)
+
+ for message in messages:
+ message["channel_info"] = channel_info
+ nmsgs += 1
+ yield message
+
+ offset += len(messages)
+
+ if offset == total:
+ fetching = False
+
+ logger.info("Fetch process completed: %s message fetched", nmsgs)
+
+ @staticmethod
+ def parse_messages(raw_messages):
+ """Parse a channel messages JSON stream.
+
+ This method parses a JSON stream, containing the
+ history of a channel. It returns a list of messages
+ and the total messages count in that channel.
+
+ :param raw_messages: JSON string to parse
+
+ :returns: a tuple with a list of dicts with the parsed messages
+ and a total messages count in the channel.
+ """
+ result = json.loads(raw_messages)
+ return result['messages'], result['total']
+
+ @staticmethod
+ def parse_channel_info(raw_channel_info):
+ """Parse a channel's information JSON stream.
+
+ This method parses a JSON stream, containing the information
+ of the channel, and returns a dict with the parsed data.
+
+ :param raw_channel_info: JSON string to parse
+
+ :returns: a dict with the parsed channel's information
+ """
+ result = json.loads(raw_channel_info)
+ return result['channel']
+
+ @classmethod
+ def has_archiving(cls):
+ """Returns whether it supports archiving items on the fetch process.
+
+ :returns: this backend supports items archive
+ """
+ return True
+
+ @classmethod
+ def has_resuming(cls):
+ """Returns whether it supports to resume the fetch process.
+
+ :returns: this backend supports items resuming
+ """
+ return True
+
+ @staticmethod
+ def metadata_id(item):
+ """Extracts the identifier from a Rocket.Chat item."""
+
+ return item["_id"]
+
+ @staticmethod
+ def metadata_updated_on(item):
+ """Extracts the update time from a Rocket.Chat item.
+
+ The timestamp is extracted from 'ts' field,
+ and then converted into a UNIX timestamp.
+
+ :param item: item generated by the backend
+
+ :returns: extracted timestamp
+ """
+ ts = str_to_datetime(item['_updatedAt']).timestamp()
+ return ts
+
+ @staticmethod
+ def metadata_category(item):
+ """Extracts the category from a Rocket.Chat item.
+
+ This backend only generates one type of item which is
+ 'message'.
+ """
+ return CATEGORY_MESSAGE
+
+ def _init_client(self, from_archive=False):
+ """Init client"""
+
+ return RocketChatClient(self.url, self.user_id, self.api_token,
+ self.max_items, self.sleep_for_rate,
+ self.min_rate_to_sleep, from_archive, self.archive, self.ssl_verify)
+
+
+class RocketChatClient(HttpClient, RateLimitHandler):
+ """Rocket.Chat API client.
+
+ Client for fetching information from the Rocket.Chat server
+ using its REST API.
+
+ :param url: server url from where messages are to be fetched
+ :param user_id: generated User Id using your Rocket.Chat account
+ :param api_token: token needed to use the API
+ :param max_items: maximum number of message requested on the same query
+ :param sleep_for_rate: sleep until rate limit is reset
+ :param min_rate_to_sleep: minimum rate needed to sleep until
+ it will be reset
+ :param from_archive: it tells whether to write/read the archive
+ :param archive: archive to store/retrieve items
+ :param ssl_verify: enable/disable SSL verification
+ """
+ RCHANNEL_MESSAGES = 'channels.messages'
+ RCHANNEL_INFO = 'channels.info'
+
+ HAUTH_TOKEN = 'X-Auth-Token'
+ HUSER_ID = 'X-User-Id'
+
+ PCHANNEL_NAME = 'roomName'
+ PCOUNT = "count"
+ POLDEST = "oldest"
+
+ def __init__(self, url, user_id, api_token, max_items=MAX_ITEMS,
+ sleep_for_rate=False, min_rate_to_sleep=MIN_RATE_LIMIT,
+ from_archive=False, archive=None, ssl_verify=True):
+
+ base_url = urijoin(url, API_EXTENSION)
+ self.user_id = user_id
+ self.api_token = api_token
+ self.max_items = max_items
+
+ super().__init__(base_url, archive=archive, from_archive=from_archive,
+ ssl_verify=ssl_verify)
+ super().setup_rate_limit_handler(sleep_for_rate=sleep_for_rate, min_rate_to_sleep=min_rate_to_sleep)
+
+ def calculate_time_to_reset(self):
+ """Number of seconds to wait. They are contained in the rate limit reset header."""
+
+ time_to_reset = self.rate_limit_reset_ts - (datetime_utcnow().replace(microsecond=0).timestamp() + 1) * 1000
+ time_to_reset /= 1000
+
+ if time_to_reset < 0:
+ time_to_reset = 0
+
+ return time_to_reset
+
+ def channel_info(self, channel):
+ """Fetch information about a channel."""
+
+ params = {
+ self.PCHANNEL_NAME: channel,
+ }
+
+ path = urijoin(self.base_url, self.RCHANNEL_INFO)
+ response = self.fetch(path, params)
+
+ return response
+
+ def messages(self, channel, from_date, offset):
+ """Fetch messages from a channel.
+
+ The messages are fetch in ascending order i.e. from the oldest
+ to the latest based on the time they were last updated. A query is
+ also passed as a param to fetch the messages from a given date.
+ """
+ query = '{"_updatedAt": {"$gte": {"$date": "%s"}}}' % from_date.isoformat()
+
+ # The 'sort' param accepts a field based on which the messages are sorted.
+ # The value of the field can be 1 for ascending order or -1 for descending order.
+ params = {
+ "roomName": channel,
+ "sort": '{"_updatedAt": 1}',
+ "count": self.max_items,
+ "offset": offset,
+ "query": query
+ }
+
+ path = urijoin(self.base_url, self.RCHANNEL_MESSAGES)
+ response = self.fetch(path, params)
+
+ return response
+
+ def fetch(self, url, payload=None, headers=None):
+ """Fetch the data from a given URL.
+
+ :param url: link to the resource
+ :param payload: payload of the request
+ :param headers: headers of the request
+
+ :returns a response object
+ """
+ headers = {
+ self.HAUTH_TOKEN: self.api_token,
+ self.HUSER_ID: self.user_id
+ }
+
+ logger.debug("Rocket.Chat client message request with params: %s", str(payload))
+
+ if not self.from_archive:
+ self.sleep_for_rate_limit()
+
+ response = super().fetch(url, payload, headers=headers)
+
+ if not self.from_archive:
+ self.update_rate_limit(response)
+
+ return response.text
+
+ @staticmethod
+ def sanitize_for_archive(url, headers, payload):
+ """Sanitize payload of a HTTP request by removing the token and
+ user id information before storing/retrieving archived items.
+
+ :param: url: HTTP url request
+ :param: headers: HTTP headers request
+ :param: payload: HTTP payload request
+
+ :returns: url, headers and the sanitized payload
+ """
+ if RocketChatClient.HAUTH_TOKEN in headers:
+ headers.pop(RocketChatClient.HAUTH_TOKEN)
+
+ if RocketChatClient.HUSER_ID in headers:
+ headers.pop(RocketChatClient.HUSER_ID)
+
+ return url, headers, payload
+
+
+class RocketChatCommand(BackendCommand):
+ """Class to run Rocket.Chat backend from the command line."""
+
+ BACKEND = RocketChat
+
+ @classmethod
+ def setup_cmd_parser(cls):
+ """Returns the Rocket.Chat argument parser."""
+
+ parser = BackendCommandArgumentParser(cls.BACKEND,
+ from_date=True,
+ token_auth=True,
+ archive=True,
+ ssl_verify=True)
+
+ # Backend token is required
+ action = parser.parser._option_string_actions['--api-token']
+ action.required = True
+
+ parser.parser.add_argument('-u', '--user-id', dest='user_id',
+ required=True,
+ help="User Id to fetch messages")
+
+ # Required positional arguments
+ parser.parser.add_argument('url',
+ help="URL of the Rocket.Chat server")
+
+ parser.parser.add_argument('channel',
+ help="Rocket.Chat channel(room) name")
+
+ # Rocket.Chat options
+ group = parser.parser.add_argument_group('Rocket.Chat arguments')
+ group.add_argument('--max-items', dest='max_items',
+ type=int, default=MAX_ITEMS,
+ help="Maximum number of items requested on the same query")
+ group.add_argument('--sleep-for-rate', dest='sleep_for_rate',
+ action='store_true',
+ help="sleep for getting more rate")
+ group.add_argument('--min-rate-to-sleep', dest='min_rate_to_sleep',
+ default=MIN_RATE_LIMIT, type=int,
+ help="sleep until reset when the rate limit reaches this value")
+
+ return parser
diff --git a/releases/unreleased/add-rocket.chat-backend.yml b/releases/unreleased/add-rocket.chat-backend.yml
new file mode 100644
index 000000000..fe6a68aed
--- /dev/null
+++ b/releases/unreleased/add-rocket.chat-backend.yml
@@ -0,0 +1,13 @@
+---
+title: Add Rocket.Chat backend
+category: added
+author: Animesh Kumar <animuz111@gmail.com>
+issue: 543
+notes: >
+ Added support to fetch messages from a Rocket.Chat
+ channel. The messages are fetched in an ascending
+ order based on time when they were last updated.
+ The channel information is also fetched.
+
+ The tests have been added accordingly.
+ The usage docs have been updated.
diff --git a/tests/data/rocketchat/channel_info.json b/tests/data/rocketchat/channel_info.json
new file mode 100644
index 000000000..ad44bcccc
--- /dev/null
+++ b/tests/data/rocketchat/channel_info.json
@@ -0,0 +1,38 @@
+{
+ "channel": {
+ "_id": "wyJHNAtuPGnQCT5xP",
+ "name": "testapichannel",
+ "fname": "testapichannel",
+ "t": "c",
+ "msgs": 3,
+ "usersCount": 2,
+ "u": {
+ "_id": "123user",
+ "username": "animesh_username1"
+ },
+ "customFields": {},
+ "broadcast": false,
+ "encrypted": false,
+ "ts": "2020-05-03T07:30:30.990Z",
+ "ro": false,
+ "default": false,
+ "sysMes": true,
+ "_updatedAt": "2020-05-03T07:32:03.594Z",
+ "lastMessage": {
+ "_id": "p5dQSb48W25EimhJK",
+ "rid": "wyJHNAtuPGnQCT5xP",
+ "msg": "Test message 2",
+ "ts": "2020-05-03T07:32:03.571Z",
+ "u": {
+ "_id": "123user",
+ "username": "animeshk_username1",
+ "name": "Animesh Kumar"
+ },
+ "_updatedAt": "2020-05-03T07:32:03.587Z",
+ "mentions": [],
+ "channels": []
+ },
+ "lm": "2020-05-03T07:32:03.571Z"
+ },
+ "success": true
+}
\ No newline at end of file
diff --git a/tests/data/rocketchat/message_empty_2020_05_10.json b/tests/data/rocketchat/message_empty_2020_05_10.json
new file mode 100644
index 000000000..cf8a847e4
--- /dev/null
+++ b/tests/data/rocketchat/message_empty_2020_05_10.json
@@ -0,0 +1,7 @@
+{
+ "messages": [],
+ "count": 0,
+ "offset": 0,
+ "total": 0,
+ "success": true
+}
\ No newline at end of file
diff --git a/tests/data/rocketchat/message_page_1.json b/tests/data/rocketchat/message_page_1.json
new file mode 100644
index 000000000..206a005cd
--- /dev/null
+++ b/tests/data/rocketchat/message_page_1.json
@@ -0,0 +1,62 @@
+{
+ "messages": [
+ {
+ "_id": "4AwA2eJQ7xBgPZ4mv",
+ "rid": "wyJHNAtuPGnQCT5xP",
+ "msg": "Test message 1",
+ "ts": "2020-05-02T07:30:42.993Z",
+ "u": {
+ "_id": "123user",
+ "username": "animesh_username1",
+ "name": "Animesh Kumar"
+ },
+ "_updatedAt": "2020-05-02T07:31:26.164Z",
+ "mentions": [],
+ "channels": [],
+ "reactions": {
+ ":cool:": {
+ "usernames": [
+ "animesh_username2"
+ ]
+ },
+ ":wait:": {
+ "usernames": [
+ "animesh_username2"
+ ]
+ }
+ },
+ "starred": [],
+ "replies": [
+ "567user"
+ ],
+ "tcount": 1,
+ "tlm": "2020-05-02T07:31:26.007Z"
+ },
+ {
+ "_id": "zofFonHMq5M3tdGyu",
+ "rid": "wyJHNAtuPGnQCT5xP",
+ "tmid": "4AwA2eJQ7xBgPZ4mv",
+ "msg": "Test reply 1",
+ "ts": "2020-05-02T07:31:26.007Z",
+ "u": {
+ "_id": "567user",
+ "username": "animesh_username2",
+ "name": "Animesh Kumar Singh"
+ },
+ "_updatedAt": "2020-05-02T07:31:56.711Z",
+ "mentions": [],
+ "channels": [],
+ "_hidden": true,
+ "parent": "WnWSwiD877xRpqcMb",
+ "editedAt": "2020-05-02T07:31:56.711Z",
+ "editedBy": {
+ "_id": "123user",
+ "username": "animesh_username1"
+ }
+ }
+ ],
+ "count": 2,
+ "offset": 0,
+ "total": 4,
+ "success": true
+}
\ No newline at end of file
diff --git a/tests/data/rocketchat/message_page_2.json b/tests/data/rocketchat/message_page_2.json
new file mode 100644
index 000000000..327e4daec
--- /dev/null
+++ b/tests/data/rocketchat/message_page_2.json
@@ -0,0 +1,43 @@
+{
+ "messages": [
+ {
+ "_id": "WnWSwiD877xRpqcMb",
+ "rid": "wyJHNAtuPGnQCT5xP",
+ "tmid": "4AwA2eJQ7xBgPZ4mv",
+ "msg": "Test reply 1 edited",
+ "ts": "2020-05-02T07:31:26.007Z",
+ "u": {
+ "_id": "567user",
+ "username": "animesh_username2",
+ "name": "Animesh Kumar Singh"
+ },
+ "_updatedAt": "2020-05-02T07:31:56.714Z",
+ "mentions": [],
+ "channels": [],
+ "editedAt": "2020-05-02T07:31:56.712Z",
+ "editedBy": {
+ "_id": "123user",
+ "username": "animesh_username1"
+ },
+ "urls": []
+ },
+ {
+ "_id": "p5dQSb48W25EimhJK",
+ "rid": "wyJHNAtuPGnQCT5xP",
+ "msg": "Test message 2",
+ "ts": "2020-05-03T07:32:03.571Z",
+ "u": {
+ "_id": "123user",
+ "username": "animesh_username1",
+ "name": "Animesh Kumar"
+ },
+ "_updatedAt": "2020-05-03T07:32:03.587Z",
+ "mentions": [],
+ "channels": []
+ }
+ ],
+ "count": 2,
+ "offset": 2,
+ "total": 4,
+ "success": true
+}
\ No newline at end of file
diff --git a/tests/data/rocketchat/message_page_2020_05_03.json b/tests/data/rocketchat/message_page_2020_05_03.json
new file mode 100644
index 000000000..f05dc3d04
--- /dev/null
+++ b/tests/data/rocketchat/message_page_2020_05_03.json
@@ -0,0 +1,22 @@
+{
+ "messages": [
+ {
+ "_id": "p5dQSb48W25EimhJK",
+ "rid": "wyJHNAtuPGnQCT5xP",
+ "msg": "Test message 2",
+ "ts": "2020-05-03T07:32:03.571Z",
+ "u": {
+ "_id": "123user",
+ "username": "animesh_username1",
+ "name": "Animesh Kumar"
+ },
+ "_updatedAt": "2020-05-03T07:32:03.587Z",
+ "mentions": [],
+ "channels": []
+ }
+ ],
+ "count": 1,
+ "offset": 0,
+ "total": 1,
+ "success": true
+}
\ No newline at end of file
diff --git a/tests/test_rocketchat.py b/tests/test_rocketchat.py
new file mode 100644
index 000000000..7ac6f4094
--- /dev/null
+++ b/tests/test_rocketchat.py
@@ -0,0 +1,564 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2015-2020 Bitergia
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Aditya Prajapati <aditya10699@gmail.com>
+# Animesh Kumar <animuz111@gmail.com>
+#
+
+import copy
+import datetime
+import httpretty
+import os
+import pkg_resources
+import unittest
+import unittest.mock
+import dateutil.tz
+import time
+import json
+
+pkg_resources.declare_namespace('perceval.backends')
+
+from perceval.backend import BackendCommandArgumentParser
+from perceval.errors import RateLimitError
+from perceval.utils import DEFAULT_DATETIME
+from perceval.backends.core.rocketchat import (RocketChat,
+ RocketChatClient,
+ RocketChatCommand,
+ MIN_RATE_LIMIT,
+ MAX_ITEMS)
+
+from base import TestCaseBackendArchive
+
+ROCKETCHAT_SERVER_URL = 'https://open.rocket.chat'
+ROCKETCHAT_CHANNEL_NAME = 'testapichannel'
+ROCKETCHAT_API_EXTENSION = "/api/v1/"
+ROCKETCHAT_API_BASE_URL = ROCKETCHAT_SERVER_URL + ROCKETCHAT_API_EXTENSION
+ROCKETCHAT_MESSAGE_URL = ROCKETCHAT_API_BASE_URL + RocketChatClient.RCHANNEL_MESSAGES
+ROCKETCHAT_CHANNEL_URL = ROCKETCHAT_API_BASE_URL + RocketChatClient.RCHANNEL_INFO
+
+
+def setup_http_server(no_message=False, rate_limit_headers=None, from_date=False):
+ """Setup a mock HTTP server"""
+
+ message_page_1 = read_file('data/rocketchat/message_page_1.json')
+ message_page_2 = read_file('data/rocketchat/message_page_2.json')
+ channel_info = read_file('data/rocketchat/channel_info.json')
+ message_empty_2020_5_10 = read_file('data/rocketchat/message_empty_2020_05_10.json')
+ message_page_2020_05_03 = read_file('data/rocketchat/message_page_2020_05_03.json')
+
+ if not rate_limit_headers:
+ rate_limit_headers = {}
+
+ httpretty.register_uri(httpretty.GET,
+ ROCKETCHAT_CHANNEL_URL + '?roomName=testapichannel',
+ body=channel_info,
+ status=200,
+ forcing_headers=rate_limit_headers)
+
+ roomName = '?roomName=testapichannel'
+ sort = '&sort={"_updatedAt": 1}'
+ count = '&count=2'
+ params = roomName + sort + count
+
+ if no_message:
+ query = '&q={"_updatedAt": {"$gte": {"$date": "2020-05-10T00:00:00+00:00"}}}'
+ httpretty.register_uri(httpretty.GET,
+ ROCKETCHAT_MESSAGE_URL + params + query + '&offset=0',
+ body=message_empty_2020_5_10,
+ status=200,
+ forcing_headers=rate_limit_headers)
+ elif from_date:
+ query = '&q={"_updatedAt": {"$gte": {"$date": "2020-05-03T00:00:00+00:00"}}}'
+ httpretty.register_uri(httpretty.GET,
+ ROCKETCHAT_MESSAGE_URL + params + query + '&offset=0',
+ body=message_page_2020_05_03,
+ status=200,
+ forcing_headers=rate_limit_headers)
+ else:
+ query = '&q={"_updatedAt": {"$gte": {"$date": "2020-05-02T00:00:00+00:00"}}}'
+ httpretty.register_uri(httpretty.GET,
+ ROCKETCHAT_MESSAGE_URL + params + query + '&offset=2',
+ body=message_page_2,
+ status=200,
+ forcing_headers=rate_limit_headers)
+
+ httpretty.register_uri(httpretty.GET,
+ ROCKETCHAT_MESSAGE_URL + params + query + '&offset=0',
+ body=message_page_1,
+ status=200,
+ forcing_headers=rate_limit_headers)
+
+
+class MockedRocketChatClient(RocketChatClient):
+ """Mocked Rocket.Chat client for testing"""
+
+ def __init__(self, url, user_id, api_token, max_items=MAX_ITEMS, archive=None,
+ sleep_for_rate=False, min_rate_to_sleep=MIN_RATE_LIMIT,
+ from_archive=False, ssl_verify=True):
+ super().__init__(url, user_id, api_token, max_items=max_items,
+ min_rate_to_sleep=min_rate_to_sleep,
+ sleep_for_rate=sleep_for_rate,
+ archive=archive,
+ from_archive=from_archive,
+ ssl_verify=ssl_verify
+ )
+ self.rate_limit_reset_ts = -1
+
+
+def read_file(filename):
+ with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), filename), 'rb') as f:
+ content = f.read()
+ return content
+
+
+class TestRocketChatBackend(unittest.TestCase):
+ """Tests for Rocket.Chat backend class"""
+
+ def test_initialization(self):
+ """Test whether attributes are initialized"""
+
+ backend = RocketChat(url='https://chat.example.com', user_id='123user', api_token='aaa',
+ channel='testapichannel', tag='test')
+
+ self.assertEqual(backend.url, 'https://chat.example.com')
+ self.assertEqual(backend.user_id, '123user')
+ self.assertEqual(backend.api_token, "aaa")
+ self.assertEqual(backend.channel, "testapichannel")
+ self.assertEqual(backend.min_rate_to_sleep, MIN_RATE_LIMIT)
+ self.assertIsNone(backend.client)
+ self.assertFalse(backend.sleep_for_rate)
+ self.assertTrue(backend.ssl_verify)
+ self.assertEqual(backend.origin, 'https://chat.example.com/testapichannel')
+ self.assertEqual(backend.tag, 'test')
+ self.assertEqual(backend.max_items, MAX_ITEMS)
+
+ # When tag is empty or None it will be set to
+ # the value in URL
+ backend = RocketChat(url="https://chat.example.com", user_id="123user", api_token='aaa',
+ channel='testapichannel', tag=None)
+ self.assertEqual(backend.origin, 'https://chat.example.com/testapichannel')
+ self.assertEqual(backend.tag, 'https://chat.example.com/testapichannel')
+
+ backend = RocketChat(url="https://chat.example.com", user_id="123user", api_token='aaa',
+ channel='testapichannel', tag='')
+ self.assertEqual(backend.origin, 'https://chat.example.com/testapichannel')
+ self.assertEqual(backend.tag, 'https://chat.example.com/testapichannel')
+
+ backend = RocketChat(url='https://chat.example.com', user_id='123user', api_token='aaa',
+ channel='testapichannel', tag='', sleep_for_rate=True,
+ ssl_verify=False, max_items=20, min_rate_to_sleep=1)
+ self.assertEqual(backend.origin, 'https://chat.example.com/testapichannel')
+ self.assertEqual(backend.tag, 'https://chat.example.com/testapichannel')
+ self.assertFalse(backend.ssl_verify)
+ self.assertTrue(backend.sleep_for_rate)
+ self.assertEqual(backend.max_items, 20)
+ self.assertEqual(backend.min_rate_to_sleep, 1)
+
+ def test_has_archiving(self):
+ """Test if it returns True when has_archiving is called"""
+
+ self.assertEqual(RocketChat.has_archiving(), True)
+
+ def test_has_resuming(self):
+ """Test if it returns True when has_resuming is called"""
+
+ self.assertEqual(RocketChat.has_resuming(), True)
+
+ @httpretty.activate
+ def test_fetch_messages(self):
+ """Test whether a list of messages is returned"""
+
+ setup_http_server()
+
+ backend = RocketChat(url='https://open.rocket.chat', user_id='123user',
+ api_token='aaa', channel='testapichannel')
+ messages = [m for m in backend.fetch()]
+
+ self.assertEqual(len(messages), 4)
+
+ message = messages[0]
+ self.assertEqual(message['data']['_id'], '4AwA2eJQ7xBgPZ4mv')
+ self.assertEqual(message['origin'], 'https://open.rocket.chat/testapichannel')
+ self.assertEqual(message['uuid'], '888b6c9a728c267435cee1d5fe8f5dbe446614a2')
+ self.assertEqual(message['updated_on'], 1588404686.164)
+ self.assertEqual(message['category'], 'message')
+ self.assertEqual(message['tag'], 'https://open.rocket.chat/testapichannel')
+ self.assertEqual(message['data']['msg'], 'Test message 1')
+ self.assertEqual(message['data']['u']['username'], 'animesh_username1')
+ self.assertEqual(message['data']['u']['name'], 'Animesh Kumar')
+ self.assertListEqual(message['data']['replies'], ["567user"])
+ self.assertEqual(message['data']['channel_info']['_id'], 'wyJHNAtuPGnQCT5xP')
+ self.assertEqual(message['data']['channel_info']['lastMessage']['msg'], 'Test message 2')
+ self.assertEqual(message['data']['channel_info']['usersCount'], 2)
+
+ message = messages[1]
+ self.assertEqual(message['data']['_id'], 'zofFonHMq5M3tdGyu')
+ self.assertEqual(message['origin'], 'https://open.rocket.chat/testapichannel')
+ self.assertEqual(message['uuid'], '6fa3f6a18491b9bcb3309f12e3a2b1cd654980c3')
+ self.assertEqual(message['updated_on'], 1588404716.711)
+ self.assertEqual(message['category'], 'message')
+ self.assertEqual(message['tag'], 'https://open.rocket.chat/testapichannel')
+ self.assertEqual(message['data']['msg'], 'Test reply 1')
+ self.assertEqual(message['data']['u']['username'], 'animesh_username2')
+ self.assertEqual(message['data']['u']['name'], 'Animesh Kumar Singh')
+ self.assertEqual(message['data']['channel_info']['_id'], 'wyJHNAtuPGnQCT5xP')
+ self.assertEqual(message['data']['channel_info']['lastMessage']['msg'], 'Test message 2')
+ self.assertEqual(message['data']['channel_info']['usersCount'], 2)
+
+ message = messages[2]
+ self.assertEqual(message['data']['_id'], 'WnWSwiD877xRpqcMb')
+ self.assertEqual(message['origin'], 'https://open.rocket.chat/testapichannel')
+ self.assertEqual(message['uuid'], '3b3afa62a63766ebeb70e3c8951c2c4b42f34767')
+ self.assertEqual(message['updated_on'], 1588404716.714)
+ self.assertEqual(message['category'], 'message')
+ self.assertEqual(message['tag'], 'https://open.rocket.chat/testapichannel')
+ self.assertEqual(message['data']['msg'], 'Test reply 1 edited')
+ self.assertEqual(message['data']['u']['username'], 'animesh_username2')
+ self.assertEqual(message['data']['u']['name'], 'Animesh Kumar Singh')
+ self.assertEqual(message['data']['editedBy']['username'], 'animesh_username1')
+ self.assertEqual(message['data']['channel_info']['_id'], 'wyJHNAtuPGnQCT5xP')
+ self.assertEqual(message['data']['channel_info']['usersCount'], 2)
+
+ message = messages[3]
+ self.assertEqual(message['data']['_id'], 'p5dQSb48W25EimhJK')
+ self.assertEqual(message['origin'], 'https://open.rocket.chat/testapichannel')
+ self.assertEqual(message['uuid'], 'a3b2a4c195e8b6a155cf5eddb2b8f79a13f836dd')
+ self.assertEqual(message['updated_on'], 1588491123.587)
+ self.assertEqual(message['category'], 'message')
+ self.assertEqual(message['tag'], 'https://open.rocket.chat/testapichannel')
+ self.assertEqual(message['data']['msg'], 'Test message 2')
+ self.assertEqual(message['data']['u']['username'], 'animesh_username1')
+ self.assertEqual(message['data']['u']['name'], 'Animesh Kumar')
+ self.assertEqual(message['data']['channel_info']['_id'], 'wyJHNAtuPGnQCT5xP')
+ self.assertEqual(message['data']['channel_info']['usersCount'], 2)
+
+ @httpretty.activate
+ def test_fetch_from_date(self):
+ """Test when fetching messages from a given date"""
+
+ setup_http_server(from_date=True)
+ from_date = datetime.datetime(2020, 5, 3, 0, 0, tzinfo=dateutil.tz.tzutc())
+
+ backend = RocketChat(url='https://open.rocket.chat', user_id='123user',
+ api_token='aaa', channel='testapichannel')
+
+ messages = [m for m in backend.fetch(from_date=from_date)]
+ self.assertEqual(len(messages), 1)
+
+ message = messages[0]
+ self.assertEqual(message['data']['_id'], 'p5dQSb48W25EimhJK')
+ self.assertEqual(message['origin'], 'https://open.rocket.chat/testapichannel')
+ self.assertEqual(message['uuid'], 'a3b2a4c195e8b6a155cf5eddb2b8f79a13f836dd')
+ self.assertEqual(message['updated_on'], 1588491123.587)
+ self.assertEqual(message['category'], 'message')
+ self.assertEqual(message['tag'], 'https://open.rocket.chat/testapichannel')
+ self.assertEqual(message['data']['msg'], 'Test message 2')
+ self.assertEqual(message['data']['u']['username'], 'animesh_username1')
+ self.assertEqual(message['data']['u']['name'], 'Animesh Kumar')
+ self.assertEqual(message['data']['channel_info']['_id'], 'wyJHNAtuPGnQCT5xP')
+ self.assertEqual(message['data']['channel_info']['usersCount'], 2)
+
+ @httpretty.activate
+ def test_search_fields_messages(self):
+ """Test whether the search_fields is properly set"""
+
+ setup_http_server()
+
+ backend = RocketChat(url='https://open.rocket.chat', user_id='123user',
+ api_token='aaa', channel='testapichannel')
+ messages = [m for m in backend.fetch()]
+
+ message = messages[0]
+ self.assertEqual(message['search_fields']['item_id'], backend.metadata_id(message['data']))
+ self.assertEqual(message['search_fields']['channel_id'], 'wyJHNAtuPGnQCT5xP')
+ self.assertEqual(message['search_fields']['channel_name'], 'testapichannel')
+
+ @httpretty.activate
+ def test_fetch_empty(self):
+ """Test whether an empty list is returned when there are no messages"""
+
+ setup_http_server(no_message=True)
+
+ backend = RocketChat(url='https://open.rocket.chat', user_id='123user',
+ api_token='aaa', channel='testapichannel')
+ messages = [m for m in backend.fetch()]
+ self.assertListEqual(messages, [])
+
+
+class TestRocketChatBackendArchive(TestCaseBackendArchive):
+ """Rocket.Chat backend tests using an archive"""
+
+ def setUp(self):
+ super().setUp()
+ self.backend_write_archive = RocketChat(url='https://open.rocket.chat', user_id='123user',
+ api_token='aaa', channel='testapichannel',
+ max_items=5, archive=self.archive)
+ self.backend_read_archive = RocketChat(url='https://open.rocket.chat', user_id='123user',
+ api_token='aaa', channel='testapichannel',
+ max_items=5, archive=self.archive)
+
+ @httpretty.activate
+ @unittest.mock.patch('perceval.backends.core.rocketchat.datetime_utcnow')
+ def test_fetch_from_archive(self, mock_utcnow):
+ """Test if a list of messages is returned from archive"""
+
+ mock_utcnow.return_value = datetime.datetime(2020, 1, 1,
+ tzinfo=dateutil.tz.tzutc())
+
+ setup_http_server()
+ self._test_fetch_from_archive(from_date=None)
+
+ @httpretty.activate
+ @unittest.mock.patch('perceval.backends.core.rocketchat.datetime_utcnow')
+ def test_fetch_from_date_from_archive(self, mock_utcnow):
+ """Test whether a list of messages is returned from archive after a given date"""
+
+ mock_utcnow.return_value = datetime.datetime(2020, 1, 1,
+ tzinfo=dateutil.tz.tzutc())
+
+ setup_http_server()
+
+ from_date = datetime.datetime(2020, 5, 3, 18, 35, 40, 69,
+ tzinfo=dateutil.tz.tzutc())
+ self._test_fetch_from_archive(from_date=from_date)
+
+ @httpretty.activate
+ @unittest.mock.patch('perceval.backends.core.rocketchat.datetime_utcnow')
+ def test_fetch_empty_from_archive(self, mock_utcnow):
+ """Test whether no messages are returned when the archive is empty"""
+
+ mock_utcnow.return_value = datetime.datetime(2020, 1, 1,
+ tzinfo=dateutil.tz.tzutc())
+
+ setup_http_server()
+
+ from_date = datetime.datetime(2020, 5, 3,
+ tzinfo=dateutil.tz.tzutc())
+ self._test_fetch_from_archive(from_date=from_date)
+
+
+class TestRocketChatClient(unittest.TestCase):
+ """Tests for RocketChatClient class"""
+
+ def test_init(self):
+ """Check attributes initialization"""
+
+ client = RocketChatClient(url='https://open.rocket.chat', user_id='123user',
+ api_token='aaa', ssl_verify=True)
+ self.assertIsInstance(client, RocketChatClient)
+ self.assertEqual(client.base_url, 'https://open.rocket.chat/api/v1')
+ self.assertEqual(client.user_id, '123user')
+ self.assertEqual(client.api_token, 'aaa')
+ self.assertTrue(client.ssl_verify)
+ self.assertFalse(client.sleep_for_rate)
+ self.assertEqual(client.min_rate_to_sleep, MIN_RATE_LIMIT)
+
+ client = RocketChatClient(url='https://open.rocket.chat', user_id='123user', api_token='aaa',
+ sleep_for_rate=True, min_rate_to_sleep=1, ssl_verify=False)
+ self.assertIsInstance(client, RocketChatClient)
+ self.assertEqual(client.base_url, 'https://open.rocket.chat/api/v1')
+ self.assertEqual(client.user_id, '123user')
+ self.assertEqual(client.api_token, 'aaa')
+ self.assertFalse(client.ssl_verify)
+ self.assertTrue(client.sleep_for_rate)
+ self.assertEqual(client.min_rate_to_sleep, 1)
+
+ @httpretty.activate
+ def test_messages(self):
+ """Test whether messages are fetched"""
+
+ setup_http_server()
+
+ client = RocketChatClient(url='https://open.rocket.chat', user_id='123user',
+ api_token='aaa', ssl_verify=True)
+
+ messages = client.messages('testapichannel', from_date=DEFAULT_DATETIME, offset=0)
+ messages = json.loads(messages)
+
+ self.assertEqual(len(messages['messages']), 2)
+
+ # Check requests
+ expected = {
+ 'count': ['100'],
+ 'offset': ['0'],
+ 'query': ['{"_updatedAt": {"$gte": {"$date": "1970-01-01T00:00:00 00:00"}}}'],
+ 'roomName': ['testapichannel'],
+ 'sort': ['{"_updatedAt": 1}']
+ }
+
+ self.assertEqual(httpretty.last_request().querystring, expected)
+ self.assertEqual(httpretty.last_request().headers[RocketChatClient.HAUTH_TOKEN], 'aaa')
+
+ @httpretty.activate
+ def test_channel_info(self):
+ """Test whether channel information is fetched"""
+
+ setup_http_server()
+
+ client = RocketChatClient(url='https://open.rocket.chat', user_id='123user',
+ api_token='aaa', ssl_verify=True)
+ channel = client.channel_info('testapichannel')
+ channel = json.loads(channel)
+
+ self.assertEqual(channel['channel']['_id'], 'wyJHNAtuPGnQCT5xP')
+ self.assertEqual(channel['channel']['name'], 'testapichannel')
+ self.assertEqual(channel['channel']['usersCount'], 2)
+ self.assertEqual(channel['channel']['msgs'], 3)
+ self.assertEqual(channel['channel']['lastMessage']['msg'], 'Test message 2')
+
+ # Check requests
+ expected = {
+ 'roomName': ['testapichannel'],
+ }
+
+ self.assertEqual(httpretty.last_request().querystring, expected)
+ self.assertEqual(httpretty.last_request().headers[RocketChatClient.HAUTH_TOKEN], 'aaa')
+
+ def test_calculate_time_to_reset(self):
+ """Test whether the time to reset is zero if the sleep time is negative"""
+
+ client = MockedRocketChatClient(url='https://open.rocket.chat', user_id='123user',
+ api_token='aaa', max_items=10, archive=None, from_archive=False,
+ min_rate_to_sleep=2, sleep_for_rate=True, ssl_verify=False)
+ time_to_reset = client.calculate_time_to_reset()
+ self.assertEqual(time_to_reset, 0)
+
+ @httpretty.activate
+ def test_sleep_for_rate(self):
+ """Test if the clients sleeps when the rate limit is reached"""
+
+ wait = 10000
+ reset = int(time.time() * 1000 + wait)
+ rate_limit_headers = {'X-RateLimit-Remaining': '0',
+ 'X-RateLimit-Reset': reset}
+
+ setup_http_server(rate_limit_headers=rate_limit_headers)
+
+ client = RocketChatClient(url='https://open.rocket.chat', user_id='123user',
+ api_token='aaa', min_rate_to_sleep=5,
+ sleep_for_rate=True)
+
+ _ = client.channel_info('testapichannel')
+ after = float(time.time() * 1000)
+
+ self.assertTrue(reset >= after)
+
+ @httpretty.activate
+ def test_rate_limit_error(self):
+ """Test if a rate limit error is raised when rate is exhausted"""
+
+ wait = 2000
+ reset = int(time.time() * 1000 + wait)
+ rate_limit_headers = {'X-RateLimit-Remaining': '0',
+ 'X-RateLimit-Reset': reset}
+
+ setup_http_server(rate_limit_headers=rate_limit_headers)
+
+ client = RocketChatClient(url='https://open.rocket.chat', user_id='123user',
+ api_token='aaa', sleep_for_rate=False)
+
+ _ = client.channel_info('testapichannel')
+ with self.assertRaises(RateLimitError):
+ _ = client.messages('testapichannel', from_date=DEFAULT_DATETIME, offset=0)
+
+ def test_sanitize_for_archive(self):
+ """Test whether the sanitize method works properly"""
+
+ url = "https://open.rocket.chat/testapichannel"
+ headers = {
+ RocketChatClient.HAUTH_TOKEN: 'aaaa',
+ RocketChatClient.HUSER_ID: '123user'
+ }
+
+ payload = {
+ 'count': 100,
+ 'offset': 0,
+ 'query': '{"_updatedAt": {"$gte": {"$date": "1970-01-01T00:00:00 00:00"}}}',
+ 'roomName': 'testapichannel',
+ 'sort': '{"_updatedAt": 1}'
+ }
+
+ s_url, s_headers, s_payload = RocketChatClient.sanitize_for_archive(url, copy.deepcopy(headers), payload)
+ headers.pop(RocketChatClient.HAUTH_TOKEN)
+ headers.pop(RocketChatClient.HUSER_ID)
+
+ self.assertEqual(url, s_url)
+ self.assertEqual(headers, s_headers)
+ self.assertEqual(payload, s_payload)
+
+
+class TestRocketChatCommand(unittest.TestCase):
+ """Tests for RocketChatCommand class"""
+
+ def test_backend_class(self):
+ """Test if the backend class is RocketChat"""
+
+ self.assertIs(RocketChatCommand.BACKEND, RocketChat)
+
+ def test_setup_cmd_parser(self):
+ """Test if it parser object is correctly initialized"""
+
+ parser = RocketChatCommand.setup_cmd_parser()
+ self.assertIsInstance(parser, BackendCommandArgumentParser)
+ self.assertEqual(parser._backend, RocketChat)
+
+ args = ['-t', 'aaa',
+ '-u', '123user',
+ '--tag', 'test',
+ '--sleep-for-rate',
+ '--from-date', '1970-01-01',
+ 'https://open.rocket.chat',
+ 'testapichannel']
+
+ parsed_args = parser.parse(*args)
+ self.assertEqual(parsed_args.api_token, 'aaa')
+ self.assertEqual(parsed_args.user_id, '123user')
+ self.assertEqual(parsed_args.url, 'https://open.rocket.chat')
+ self.assertEqual(parsed_args.channel, 'testapichannel')
+ self.assertEqual(parsed_args.tag, 'test')
+ self.assertEqual(parsed_args.from_date, DEFAULT_DATETIME)
+ self.assertTrue(parsed_args.ssl_verify)
+ self.assertTrue(parsed_args.sleep_for_rate)
+
+ from_date = datetime.datetime(2020, 3, 1, 0, 0, tzinfo=dateutil.tz.tzutc())
+
+ args = ['-t', 'aaa',
+ '-u', '123user',
+ '--tag', 'test',
+ '--max-items', '10',
+ '--no-ssl-verify',
+ '--min-rate-to-sleep', '1',
+ '--from-date', '2020-03-01',
+ 'https://open.rocket.chat',
+ 'testapichannel']
+
+ parsed_args = parser.parse(*args)
+ self.assertEqual(parsed_args.api_token, 'aaa')
+ self.assertEqual(parsed_args.user_id, '123user')
+ self.assertEqual(parsed_args.url, 'https://open.rocket.chat')
+ self.assertEqual(parsed_args.channel, 'testapichannel')
+ self.assertEqual(parsed_args.tag, 'test')
+ self.assertEqual(parsed_args.from_date, from_date)
+ self.assertEqual(parsed_args.min_rate_to_sleep, 1)
+ self.assertFalse(parsed_args.ssl_verify)
+ self.assertFalse(parsed_args.sleep_for_rate)
+
+
+if __name__ == "__main__":
+ unittest.main(warnings='ignore')