From 1a746e2672ae9300efec0625fc07f61694f1c400 Mon Sep 17 00:00:00 2001 From: Valerio Cosentino Date: Tue, 25 Feb 2020 19:40:39 +0100 Subject: [PATCH] [elk] Remove support for arthur This code removes the references, methods depedencies and config params used to fetch data from arthur. This change is needed since the integration with arthur will be implemented in a different way (as discussed in the roadmap). Signed-off-by: Valerio Cosentino --- grimoire_elk/elk.py | 95 ++++++------------------------- grimoire_elk/enriched/enrich.py | 1 - grimoire_elk/raw/dockerhub.py | 10 ---- grimoire_elk/raw/elastic.py | 13 +---- grimoire_elk/raw/github.py | 20 ++----- grimoire_elk/raw/gitlab.py | 19 ------- grimoire_elk/raw/google_hits.py | 5 +- grimoire_elk/raw/groupsio.py | 7 --- grimoire_elk/raw/jira.py | 8 --- grimoire_elk/raw/mattermost.py | 15 ++--- grimoire_elk/raw/mbox.py | 10 ---- grimoire_elk/raw/mediawiki.py | 13 +---- grimoire_elk/raw/meetup.py | 8 --- grimoire_elk/raw/nntp.py | 10 ---- grimoire_elk/raw/pipermail.py | 7 --- grimoire_elk/raw/slack.py | 8 --- grimoire_elk/raw/stackexchange.py | 12 ---- grimoire_elk/raw/supybot.py | 10 ---- grimoire_elk/raw/telegram.py | 8 +-- grimoire_elk/raw/twitter.py | 8 --- grimoire_elk/utils.py | 13 ++--- requirements.txt | 2 - setup.py | 2 - tests/test_askbot.py | 11 ---- tests/test_bugzilla.py | 10 ---- tests/test_bugzillarest.py | 9 --- tests/test_confluence.py | 10 ---- tests/test_crates.py | 10 ---- tests/test_discourse.py | 10 ---- tests/test_dockerhub.py | 10 ---- tests/test_functest.py | 10 ---- tests/test_gerrit.py | 10 ---- tests/test_git.py | 10 ---- tests/test_github.py | 10 ---- tests/test_github2.py | 10 ---- tests/test_gitlab.py | 16 ------ tests/test_google_hits.py | 10 ---- tests/test_groupsio.py | 10 ---- tests/test_hyperkitty.py | 11 ---- tests/test_jenkins.py | 10 ---- tests/test_jira.py | 8 --- tests/test_kitsune.py | 10 ---- tests/test_mattermost.py | 10 ---- tests/test_mbox.py | 10 ---- tests/test_mediawiki.py | 9 --- tests/test_meetup.py | 9 --- tests/test_mozillaclub.py | 12 ---- tests/test_nntp.py | 10 ---- tests/test_phabricator.py | 10 ---- tests/test_pipermail.py | 10 ---- tests/test_puppetforge.py | 10 ---- tests/test_redmine.py | 10 ---- tests/test_remo.py | 10 ---- tests/test_rss.py | 10 ---- tests/test_slack.py | 10 ---- tests/test_stackexchange.py | 10 ---- tests/test_supybot.py | 10 ---- tests/test_telegram.py | 10 ---- tests/test_twitter.py | 10 ---- utils/p2o.py | 3 +- 60 files changed, 38 insertions(+), 634 deletions(-) diff --git a/grimoire_elk/elk.py b/grimoire_elk/elk.py index 4f6ae35c8..940615fb1 100755 --- a/grimoire_elk/elk.py +++ b/grimoire_elk/elk.py @@ -21,15 +21,12 @@ import inspect import logging -import pickle -import redis from elasticsearch import Elasticsearch from datetime import datetime from dateutil import parser -from arthur.common import Q_STORAGE_ITEMS from perceval.backend import find_signature_parameters, Archive from perceval.errors import RateLimitError from grimoirelab_toolkit.datetime import datetime_utcnow @@ -48,62 +45,9 @@ requests_ses = grimoire_con() -arthur_items = {} # Hash with tag list with all items collected from arthur queue - - -def feed_arthur(): - """ Feed Ocean with backend data collected from arthur redis queue""" - - logger.info("Collecting items from redis queue") - - db_url = 'redis://localhost/8' - - conn = redis.StrictRedis.from_url(db_url) - logger.debug("Redis connection stablished with {}.".format(db_url)) - - # Get and remove queued items in an atomic transaction - pipe = conn.pipeline() - pipe.lrange(Q_STORAGE_ITEMS, 0, -1) - pipe.ltrim(Q_STORAGE_ITEMS, 1, 0) - items = pipe.execute()[0] - - for item in items: - arthur_item = pickle.loads(item) - if arthur_item['tag'] not in arthur_items: - arthur_items[arthur_item['tag']] = [] - arthur_items[arthur_item['tag']].append(arthur_item) - - for tag in arthur_items: - logger.debug("Items for {}: {}".format(tag, len(arthur_items[tag]))) - - -def feed_backend_arthur(backend_name, backend_params): - """ Feed Ocean with backend data collected from arthur redis queue""" - - # Always get pending items from arthur for all data sources - feed_arthur() - - logger.debug("Items available for {}".format(arthur_items.keys())) - - # Get only the items for the backend - if not get_connector_from_name(backend_name): - raise RuntimeError("Unknown backend {}".format(backend_name)) - connector = get_connector_from_name(backend_name) - klass = connector[3] # BackendCmd for the connector - - backend_cmd = init_backend(klass(*backend_params)) - - tag = backend_cmd.backend.tag - logger.debug("Getting items for {}.".format(tag)) - - if tag in arthur_items: - logger.debug("Found items for {}.".format(tag)) - for item in arthur_items[tag]: - yield item - def feed_backend(url, clean, fetch_archive, backend_name, backend_params, - es_index=None, es_index_enrich=None, project=None, arthur=False, + es_index=None, es_index_enrich=None, project=None, es_aliases=None, projects_json_repo=None, repo_labels=None): """ Feed Ocean with backend data """ @@ -204,28 +148,21 @@ def feed_backend(url, clean, fetch_archive, backend_name, backend_params, except AttributeError: latest_items = backend_cmd.parsed_args.latest_items - # fetch params support - if arthur: - # If using arthur just provide the items generator to be used - # to collect the items and upload to Elasticsearch - aitems = feed_backend_arthur(backend_name, backend_params) - ocean_backend.feed(arthur_items=aitems) - else: - params = {} - if latest_items: - params['latest_items'] = latest_items - if category: - params['category'] = category - if branches: - params['branches'] = branches - if filter_classified: - params['filter_classified'] = filter_classified - if from_date and (from_date.replace(tzinfo=None) != parser.parse("1970-01-01")): - params['from_date'] = from_date - if offset: - params['from_offset'] = offset - - ocean_backend.feed(**params) + params = {} + if latest_items: + params['latest_items'] = latest_items + if category: + params['category'] = category + if branches: + params['branches'] = branches + if filter_classified: + params['filter_classified'] = filter_classified + if from_date and (from_date.replace(tzinfo=None) != parser.parse("1970-01-01")): + params['from_date'] = from_date + if offset: + params['from_offset'] = offset + + ocean_backend.feed(**params) except RateLimitError as ex: logger.error("Error feeding raw from {} ({}): rate limit exceeded".format(backend_name, backend.origin)) diff --git a/grimoire_elk/enriched/enrich.py b/grimoire_elk/enriched/enrich.py index 13ee71702..cc199317c 100644 --- a/grimoire_elk/enriched/enrich.py +++ b/grimoire_elk/enriched/enrich.py @@ -159,7 +159,6 @@ def __init__(self, db_sortinghat=None, db_projects_map=None, json_projects_map=N # params used to configure the backend # in perceval backends managed directly inside the backend - # in twitter and others managed in arthur logic self.backend_params = None # Label used during enrichment for identities without a known affiliation self.unaffiliated_group = 'Unknown' diff --git a/grimoire_elk/raw/dockerhub.py b/grimoire_elk/raw/dockerhub.py index b515fd1c7..34c9b0053 100644 --- a/grimoire_elk/raw/dockerhub.py +++ b/grimoire_elk/raw/dockerhub.py @@ -67,13 +67,3 @@ def get_perceval_params_from_url(cls, url): params = url.split() return params - - @classmethod - def get_arthur_params_from_url(cls, url): - # In the url the org and the repository are included - - params = url.split() - """ Get the arthur params given a URL for the data source """ - params = {"owner": params[0], "repository": params[1]} - - return params diff --git a/grimoire_elk/raw/elastic.py b/grimoire_elk/raw/elastic.py index 7d088c67b..522b5876b 100644 --- a/grimoire_elk/raw/elastic.py +++ b/grimoire_elk/raw/elastic.py @@ -133,11 +133,6 @@ def get_perceval_params_from_url(cls, url): """ Get the perceval params given a URL for the data source """ return [url] - @classmethod - def get_arthur_params_from_url(cls, url): - """ Get the arthur params given a URL for the data source """ - return {"uri": url, "url": url} - def drop_item(self, item): """ Drop items not to be inserted in Elastic """ return False @@ -155,17 +150,13 @@ def add_update_date(self, item): item['metadata__timestamp'] = timestamp.isoformat() def feed(self, from_date=None, from_offset=None, category=None, branches=None, - latest_items=None, arthur_items=None, filter_classified=None): - """ Feed data in Elastic from Perceval or Arthur """ + latest_items=None, filter_classified=None): + """Feed data in Elastic from Perceval""" if self.fetch_archive: items = self.perceval_backend.fetch_from_archive() self.feed_items(items) return - elif arthur_items: - items = arthur_items - self.feed_items(items) - return if from_date and from_offset: raise RuntimeError("Can't not feed using from_date and from_offset.") diff --git a/grimoire_elk/raw/github.py b/grimoire_elk/raw/github.py index 92cc7aa04..5d2db0f01 100644 --- a/grimoire_elk/raw/github.py +++ b/grimoire_elk/raw/github.py @@ -88,27 +88,15 @@ class GitHubOcean(ElasticOcean): mapping = Mapping - @classmethod - def get_arthur_params_from_url(cls, url): - """ Get the arthur params given a URL for the data source """ - params = {} - - owner = url.split('/')[-2] - repository = url.split('/')[-1] - # params.append('--owner') - params['owner'] = owner - # params.append('--repository') - params['repository'] = repository - return params - @classmethod def get_perceval_params_from_url(cls, url): """ Get the perceval params given a URL for the data source """ params = [] - dparam = cls.get_arthur_params_from_url(url) - params.append(dparam['owner']) - params.append(dparam['repository']) + owner = url.split('/')[-2] + repository = url.split('/')[-1] + params.append(owner) + params.append(repository) return params def _fix_item(self, item): diff --git a/grimoire_elk/raw/gitlab.py b/grimoire_elk/raw/gitlab.py index c827f9cf1..5d4a9a13e 100644 --- a/grimoire_elk/raw/gitlab.py +++ b/grimoire_elk/raw/gitlab.py @@ -19,8 +19,6 @@ # Valerio Cosentino # -from perceval.backends.core.gitlab import GitLabCommand - from .elastic import ElasticOcean from ..elastic_mapping import Mapping as BaseMapping @@ -72,23 +70,6 @@ class GitLabOcean(ElasticOcean): mapping = Mapping - @classmethod - def get_arthur_params_from_url(cls, url): - """ Get the arthur params given a URL for the data source """ - params = {} - - args = cls.get_perceval_params_from_url(url) - parser = GitLabCommand.setup_cmd_parser() - - parsed_args = parser.parse(*args) - - params['owner'] = parsed_args.owner - params['repository'] = parsed_args.repository - # include only blacklist ids information - params['blacklist_ids'] = parsed_args.blacklist_ids - - return params - @classmethod def get_perceval_params_from_url(cls, url): """ Get the perceval params given a URL for the data source """ diff --git a/grimoire_elk/raw/google_hits.py b/grimoire_elk/raw/google_hits.py index aaf4dd0f5..8d52cf1bd 100644 --- a/grimoire_elk/raw/google_hits.py +++ b/grimoire_elk/raw/google_hits.py @@ -27,7 +27,4 @@ class GoogleHitsOcean(ElasticOcean): """GoogleHits Ocean feeder""" - @classmethod - def get_arthur_params_from_url(cls, url): - """ Get the arthur params given a URL for the data source """ - return url + pass diff --git a/grimoire_elk/raw/groupsio.py b/grimoire_elk/raw/groupsio.py index c5a9a21ff..5f7485584 100644 --- a/grimoire_elk/raw/groupsio.py +++ b/grimoire_elk/raw/groupsio.py @@ -26,10 +26,3 @@ class GroupsioOcean(MBoxOcean): """Groups.io Ocean feeder""" mapping = Mapping - - @classmethod - def get_arthur_params_from_url(cls, url): - # In the url the uri and the data dir are included - params = url.split() - - return {"group_name": params[0], "dirpath": "/tmp"} diff --git a/grimoire_elk/raw/jira.py b/grimoire_elk/raw/jira.py index 0c91b188f..e1f3760da 100644 --- a/grimoire_elk/raw/jira.py +++ b/grimoire_elk/raw/jira.py @@ -93,14 +93,6 @@ class JiraOcean(ElasticOcean): mapping = Mapping - @classmethod - def get_arthur_params_from_url(cls, url): - """ Get the arthur params given a URL for the data source """ - - tokens = url.split(' ', 1) - - return {"url": tokens[0]} - def _fix_item(self, item): # Remove all custom fields to avoid the 1000 fields limit in ES diff --git a/grimoire_elk/raw/mattermost.py b/grimoire_elk/raw/mattermost.py index 3361def84..9b83e8a0f 100644 --- a/grimoire_elk/raw/mattermost.py +++ b/grimoire_elk/raw/mattermost.py @@ -77,16 +77,9 @@ def get_perceval_params_from_url(cls, url): """ Get the perceval params given a URL for the data source """ params = [] - dparam = cls.get_arthur_params_from_url(url) - params.append(dparam['url']) - params.append(dparam['channel']) - return params - - @classmethod - def get_arthur_params_from_url(cls, url): - # The URL is the mattermost URL and the [id of the channel data = url.split() - - params = {"url": data[0], "channel": data[1]} - + url = data[0] + channel = data[1] + params.append(url) + params.append(channel) return params diff --git a/grimoire_elk/raw/mbox.py b/grimoire_elk/raw/mbox.py index 15ce2dc71..479498e14 100644 --- a/grimoire_elk/raw/mbox.py +++ b/grimoire_elk/raw/mbox.py @@ -72,16 +72,6 @@ def get_perceval_params_from_url(cls, url): return params - @classmethod - def get_arthur_params_from_url(cls, url): - # In the url the dirpath and the repository are included - - params = url.split() - """ Get the arthur params given a URL for the data source """ - params = {"dirpath": params[1], "uri": params[0]} - - return params - def _fix_item(self, item): # Remove all custom fields to avoid the 1000 fields limit in ES diff --git a/grimoire_elk/raw/mediawiki.py b/grimoire_elk/raw/mediawiki.py index 380a99be3..7b236b799 100644 --- a/grimoire_elk/raw/mediawiki.py +++ b/grimoire_elk/raw/mediawiki.py @@ -66,15 +66,8 @@ def get_perceval_params_from_url(cls, urls): """ Get the perceval params given the URLs for the data source """ params = [] - dparam = cls.get_arthur_params_from_url(urls) - params.append(dparam["url"]) - - return params - - @classmethod - def get_arthur_params_from_url(cls, urls): - # The URLs are the API URL and the wiki URL - # We just need the API URL data = urls.split() + url = data[0] + params.append(url) - return {"url": data[0]} + return params diff --git a/grimoire_elk/raw/meetup.py b/grimoire_elk/raw/meetup.py index 2f01e1214..610813f28 100644 --- a/grimoire_elk/raw/meetup.py +++ b/grimoire_elk/raw/meetup.py @@ -80,11 +80,3 @@ def get_perceval_params_from_url(cls, url): params.append(url) return params - - @classmethod - def get_arthur_params_from_url(cls, url): - # The URL is directly the meetup group - - params = {"group": url} - - return params diff --git a/grimoire_elk/raw/nntp.py b/grimoire_elk/raw/nntp.py index fcc3b0cd8..0136692bb 100644 --- a/grimoire_elk/raw/nntp.py +++ b/grimoire_elk/raw/nntp.py @@ -63,16 +63,6 @@ def get_perceval_params_from_url(cls, url): return params - @classmethod - def get_arthur_params_from_url(cls, url): - # In the url the NNTP host and the group are included - - params = url.split() - """ Get the arthur params given a URL for the data source """ - params = {"host": params[0], "group": params[1]} - - return params - def _fix_item(self, item): # Remove all custom fields to avoid the 1000 fields limit in ES diff --git a/grimoire_elk/raw/pipermail.py b/grimoire_elk/raw/pipermail.py index 8ee9d5dea..d8f71c802 100644 --- a/grimoire_elk/raw/pipermail.py +++ b/grimoire_elk/raw/pipermail.py @@ -26,10 +26,3 @@ class PipermailOcean(MBoxOcean): """Pipermail Ocean feeder""" mapping = Mapping - - @classmethod - def get_arthur_params_from_url(cls, url): - # In the url the uri and the data dir are included - params = url.split() - - return {"url": params[0], "dirpath": "/tmp"} diff --git a/grimoire_elk/raw/slack.py b/grimoire_elk/raw/slack.py index 884853c48..d73972b4e 100644 --- a/grimoire_elk/raw/slack.py +++ b/grimoire_elk/raw/slack.py @@ -76,14 +76,6 @@ class SlackOcean(ElasticOcean): mapping = Mapping - @classmethod - def get_arthur_params_from_url(cls, url): - # The URL is directly the slack group - - params = {"channel": url} - - return params - def _fix_item(self, item): if 'channel_info' in item['data']: item['data']['channel_info'].pop('previous_names', None) diff --git a/grimoire_elk/raw/stackexchange.py b/grimoire_elk/raw/stackexchange.py index 82d7c291a..93fcaeca3 100644 --- a/grimoire_elk/raw/stackexchange.py +++ b/grimoire_elk/raw/stackexchange.py @@ -80,15 +80,3 @@ def get_perceval_params_from_url(cls, url): params.append(url) return params - - @classmethod - def get_arthur_params_from_url(cls, url): - params = [] - - tokens = url.replace('https://', '').replace('http://', '').split('/') - tag = tokens[-1] - site = tokens[0] - - params = {"site": site, "tagged": tag} - - return params diff --git a/grimoire_elk/raw/supybot.py b/grimoire_elk/raw/supybot.py index 0a38e9bd9..2845847c1 100644 --- a/grimoire_elk/raw/supybot.py +++ b/grimoire_elk/raw/supybot.py @@ -31,13 +31,3 @@ def get_perceval_params_from_url(cls, url): params = url.split() return params - - @classmethod - def get_arthur_params_from_url(cls, url): - # In the url the uri and the dirpath are included - - params = url.split() - """ Get the arthur params given a URL for the data source """ - params = {"uri": params[0], "dirpath": params[1]} - - return params diff --git a/grimoire_elk/raw/telegram.py b/grimoire_elk/raw/telegram.py index ba39cb23a..247871776 100644 --- a/grimoire_elk/raw/telegram.py +++ b/grimoire_elk/raw/telegram.py @@ -25,10 +25,4 @@ class TelegramOcean(ElasticOcean): """Telegram Ocean feeder""" - @classmethod - def get_arthur_params_from_url(cls, url): - # The URL is directly the name of the bot - - params = {"bot": url} - - return params + pass diff --git a/grimoire_elk/raw/twitter.py b/grimoire_elk/raw/twitter.py index e0e19c45b..0e585b983 100644 --- a/grimoire_elk/raw/twitter.py +++ b/grimoire_elk/raw/twitter.py @@ -108,11 +108,3 @@ class TwitterOcean(ElasticOcean): """Twitter Ocean feeder""" mapping = Mapping - - @classmethod - def get_arthur_params_from_url(cls, query): - # The query is directly passed to the Twitter API - - params = {"query": query} - - return params diff --git a/grimoire_elk/utils.py b/grimoire_elk/utils.py index cbfa51929..367e3d085 100755 --- a/grimoire_elk/utils.py +++ b/grimoire_elk/utils.py @@ -313,17 +313,17 @@ def config_logging(debug): logging.getLogger("perceval.backends.core.git").setLevel(logging.WARNING) -ARTHUR_USAGE_MSG = '' -ARTHUR_DESC_MSG = '' -ARTHUR_EPILOG_MSG = '' +USAGE_MSG = '' +DESC_MSG = '' +EPILOG_MSG = '' def get_params_parser(): """Parse command line arguments""" - parser = argparse.ArgumentParser(usage=ARTHUR_USAGE_MSG, - description=ARTHUR_DESC_MSG, - epilog=ARTHUR_EPILOG_MSG, + parser = argparse.ArgumentParser(usage=USAGE_MSG, + description=DESC_MSG, + epilog=EPILOG_MSG, formatter_class=argparse.RawDescriptionHelpFormatter, add_help=False) @@ -374,7 +374,6 @@ def get_params_parser(): parser.add_argument('--scroll-wait', default=900, type=int, help="Wait for available scroll (default 900s)") parser.add_argument('--scroll-size', default=100, type=int, help="Number of items to get from Elasticsearch when scrolling.") - parser.add_argument('--arthur', action='store_true', help="Read items from arthur redis queue") parser.add_argument('--pair-programming', action='store_true', help="Do pair programming in git enrich") parser.add_argument('--studies-list', nargs='*', help="List of studies to be executed") parser.add_argument('backend', help=argparse.SUPPRESS) diff --git a/requirements.txt b/requirements.txt index d1514fd46..6bbf39bee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,14 +3,12 @@ elasticsearch-dsl==6.3.1 requests==2.21.0 urllib3==1.24.3 PyMySQL>=0.7.0 -redis==3.0.0 geopy>=1.20.0 pandas>=0.22.0,<=0.25.3 statsmodels >= 0.9.0 -e git+https://github.com/chaoss/grimoirelab-sortinghat/#egg=grimoirelab-sortinghat -e git+https://github.com/chaoss/grimoirelab-toolkit/#egg=grimoirelab-toolkit -e git+https://github.com/chaoss/grimoirelab-cereslib/#egg=grimoirelab-cereslib --e git+https://github.com/chaoss/grimoirelab-kingarthur/#egg=grimoirelab-kingarthur -e git+https://github.com/chaoss/grimoirelab-graal/#egg=grimoirelab-graal -e git+https://github.com/chaoss/grimoirelab-perceval/#egg=grimoirelab-perceval -e git+https://github.com/chaoss/grimoirelab-perceval-mozilla/#egg=grimoirelab-perceval-mozilla diff --git a/setup.py b/setup.py index 4dc31103a..22ca17aa8 100755 --- a/setup.py +++ b/setup.py @@ -72,7 +72,6 @@ 'perceval-opnfv>=0.1.2', 'perceval-puppet>=0.1.4', 'perceval-finos>=0.1.0', - 'kingarthur>=0.1.1', 'cereslib>=0.1.0', 'grimoirelab-toolkit>=0.1.4', 'sortinghat>=0.6.2', @@ -82,7 +81,6 @@ 'requests==2.21.0', 'urllib3==1.24.3', 'PyMySQL>=0.7.0', - 'redis==3.0.0', 'pandas>=0.22.0,<=0.25.3', 'geopy>=1.20.0', 'statsmodels >= 0.9.0' diff --git a/tests/test_askbot.py b/tests/test_askbot.py index 53474db6f..09b9ec928 100644 --- a/tests/test_askbot.py +++ b/tests/test_askbot.py @@ -19,13 +19,11 @@ # Alvaro del Castillo # Valerio Cosentino # -import json import logging import unittest from base import TestBaseBackend from grimoire_elk.enriched.utils import REPO_LABELS -from grimoire_elk.raw.askbot import AskbotOcean class TestAskbot(TestBaseBackend): @@ -92,15 +90,6 @@ def test_refresh_project(self): result = self._test_refresh_project() # .. ? - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - with open("data/projects-release.json") as projects_filename: - url = json.load(projects_filename)['grimoire']['askbot'][0] - print(AskbotOcean.get_arthur_params_from_url(url)) - arthur_params = {'uri': 'https://ask.puppet.com', 'url': 'https://ask.puppet.com'} - self.assertDictEqual(arthur_params, AskbotOcean.get_arthur_params_from_url(url)) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_bugzilla.py b/tests/test_bugzilla.py index 5e6b3691e..cbc7b0a3c 100644 --- a/tests/test_bugzilla.py +++ b/tests/test_bugzilla.py @@ -19,13 +19,11 @@ # Alvaro del Castillo # Valerio Cosentino # -import json import logging import unittest from base import TestBaseBackend from grimoire_elk.enriched.utils import REPO_LABELS -from grimoire_elk.raw.bugzilla import BugzillaOcean class TestBugzilla(TestBaseBackend): @@ -99,14 +97,6 @@ def test_refresh_project(self): result = self._test_refresh_project() # ... ? - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - with open("data/projects-release.json") as projects_filename: - bugzilla_url = json.load(projects_filename)['grimoire']['bugzilla'][0] - arthur_params = {'uri': 'https://bugs.eclipse.org/bugs/', 'url': 'https://bugs.eclipse.org/bugs/'} - self.assertDictEqual(arthur_params, BugzillaOcean.get_arthur_params_from_url(bugzilla_url)) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_bugzillarest.py b/tests/test_bugzillarest.py index 7e7c50fb6..34dc5619c 100644 --- a/tests/test_bugzillarest.py +++ b/tests/test_bugzillarest.py @@ -26,7 +26,6 @@ from base import TestBaseBackend from grimoire_elk.enriched.utils import REPO_LABELS -from grimoire_elk.raw.bugzillarest import BugzillaRESTOcean from grimoire_elk.enriched.bugzillarest import BugzillaRESTEnrich @@ -130,14 +129,6 @@ def test_get_project(self): # Check the project self.assertEqual(eitem['project'], "Localization") - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - with open("data/projects-release.json") as projects_filename: - url = json.load(projects_filename)['grimoire']['bugzillarest'][0] - arthur_params = {'uri': 'https://bugzilla.mozilla.org', 'url': 'https://bugzilla.mozilla.org'} - self.assertDictEqual(arthur_params, BugzillaRESTOcean.get_arthur_params_from_url(url)) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_confluence.py b/tests/test_confluence.py index fa281ea64..efec9baf6 100644 --- a/tests/test_confluence.py +++ b/tests/test_confluence.py @@ -19,13 +19,11 @@ # Alvaro del Castillo # Valerio Cosentino # -import json import logging import unittest from base import TestBaseBackend from grimoire_elk.enriched.utils import REPO_LABELS -from grimoire_elk.raw.confluence import ConfluenceOcean from grimoire_elk.enriched.confluence import NO_ANCESTOR_TITLE @@ -120,14 +118,6 @@ def test_refresh_project(self): result = self._test_refresh_project() # ... ? - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - with open("data/projects-release.json") as projects_filename: - url = json.load(projects_filename)['grimoire']['confluence'][0] - arthur_params = {'uri': 'https://wiki.open-o.org/', 'url': 'https://wiki.open-o.org/'} - self.assertDictEqual(arthur_params, ConfluenceOcean.get_arthur_params_from_url(url)) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_crates.py b/tests/test_crates.py index cd92c4bb3..38f9ea671 100644 --- a/tests/test_crates.py +++ b/tests/test_crates.py @@ -99,16 +99,6 @@ def test_perceval_params(self): expected_params = [] self.assertListEqual(CratesOcean.get_perceval_params_from_url(url), expected_params) - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - url = "" - expected_params = { - 'url': '', - 'uri': '' - } - self.assertDictEqual(CratesOcean.get_arthur_params_from_url(url), expected_params) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_discourse.py b/tests/test_discourse.py index d40d35172..1100557f4 100644 --- a/tests/test_discourse.py +++ b/tests/test_discourse.py @@ -19,13 +19,11 @@ # Alvaro del Castillo # Valerio Cosentino # -import json import logging import unittest from base import TestBaseBackend from grimoire_elk.enriched.utils import REPO_LABELS -from grimoire_elk.raw.discourse import DiscourseOcean class TestDiscourse(TestBaseBackend): @@ -98,14 +96,6 @@ def test_refresh_project(self): result = self._test_refresh_project() # ... ? - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - with open("data/projects-release.json") as projects_filename: - url = json.load(projects_filename)['grimoire']['discourse'][0] - arthur_params = {'uri': 'https://forum.mautic.org/', 'url': 'https://forum.mautic.org/'} - self.assertDictEqual(arthur_params, DiscourseOcean.get_arthur_params_from_url(url)) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_dockerhub.py b/tests/test_dockerhub.py index 3c5a95719..46eaeb3a5 100644 --- a/tests/test_dockerhub.py +++ b/tests/test_dockerhub.py @@ -105,16 +105,6 @@ def test_perceval_params(self): ] self.assertListEqual(DockerHubOcean.get_perceval_params_from_url(url), expected_params) - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - url = "bitergia kibiter" - expected_params = { - 'owner': 'bitergia', - 'repository': 'kibiter' - } - self.assertDictEqual(DockerHubOcean.get_arthur_params_from_url(url), expected_params) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_functest.py b/tests/test_functest.py index b053883e2..0893d32d1 100644 --- a/tests/test_functest.py +++ b/tests/test_functest.py @@ -19,13 +19,11 @@ # Alvaro del Castillo # Valerio Cosentino # -import json import logging import unittest from base import TestBaseBackend from grimoire_elk.enriched.utils import REPO_LABELS -from grimoire_elk.raw.functest import FunctestOcean class TestFunctest(TestBaseBackend): @@ -118,14 +116,6 @@ def test_refresh_project(self): result = self._test_refresh_project() # ... ? - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - with open("data/projects-release.json") as projects_filename: - url = json.load(projects_filename)['grimoire']['functest'][0] - arthur_params = {'uri': 'http://testresults.opnfv.org/test/', 'url': 'http://testresults.opnfv.org/test/'} - self.assertDictEqual(arthur_params, FunctestOcean.get_arthur_params_from_url(url)) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_gerrit.py b/tests/test_gerrit.py index 5dc9d632e..1eee48f12 100644 --- a/tests/test_gerrit.py +++ b/tests/test_gerrit.py @@ -19,13 +19,11 @@ # Alvaro del Castillo # Valerio Cosentino # -import json import logging import time import unittest from base import TestBaseBackend, DB_SORTINGHAT -from grimoire_elk.raw.gerrit import GerritOcean from grimoire_elk.enriched.enrich import (logger, DEMOGRAPHICS_ALIAS) from grimoire_elk.enriched.utils import REPO_LABELS @@ -301,14 +299,6 @@ def test_refresh_project(self): result = self._test_refresh_project() # ... ? - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - with open("data/projects-release.json") as projects_filename: - url = json.load(projects_filename)['grimoire']['gerrit'][0] - arthur_params = {'uri': 'review.openstack.org', 'url': 'review.openstack.org'} - self.assertDictEqual(arthur_params, GerritOcean.get_arthur_params_from_url(url)) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_git.py b/tests/test_git.py index 8084a6140..98fc9a096 100644 --- a/tests/test_git.py +++ b/tests/test_git.py @@ -284,16 +284,6 @@ def test_perceval_params(self): ] self.assertListEqual(GitOcean.get_perceval_params_from_url(url), expected_params) - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - url = "https://github.com/grimoirelab/perceval" - expected_params = { - 'uri': 'https://github.com/grimoirelab/perceval', - 'url': 'https://github.com/grimoirelab/perceval' - } - self.assertDictEqual(GitOcean.get_arthur_params_from_url(url), expected_params) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_github.py b/tests/test_github.py index c17fa1b91..da70293bd 100644 --- a/tests/test_github.py +++ b/tests/test_github.py @@ -163,16 +163,6 @@ def test_perceval_params(self): ] self.assertListEqual(GitHubOcean.get_perceval_params_from_url(url), expected_params) - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - url = "https://github.com/chaoss/grimoirelab-perceval" - expected_params = { - 'owner': 'chaoss', - 'repository': 'grimoirelab-perceval' - } - self.assertDictEqual(GitHubOcean.get_arthur_params_from_url(url), expected_params) - def test_geolocation_study(self): """ Test that the geolocation study works correctly """ diff --git a/tests/test_github2.py b/tests/test_github2.py index 5611c0e73..b5eef1dca 100644 --- a/tests/test_github2.py +++ b/tests/test_github2.py @@ -181,16 +181,6 @@ def test_perceval_params(self): ] self.assertListEqual(GitHubOcean.get_perceval_params_from_url(url), expected_params) - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - url = "https://github.com/chaoss/grimoirelab-perceval" - expected_params = { - 'owner': 'chaoss', - 'repository': 'grimoirelab-perceval' - } - self.assertDictEqual(GitHubOcean.get_arthur_params_from_url(url), expected_params) - def test_geolocation_study(self): """ Test that the geolocation study works correctly """ diff --git a/tests/test_gitlab.py b/tests/test_gitlab.py index b0c657178..25150e760 100644 --- a/tests/test_gitlab.py +++ b/tests/test_gitlab.py @@ -22,7 +22,6 @@ import unittest from base import TestBaseBackend -from grimoire_elk.raw.gitlab import GitLabOcean from grimoire_elk.enriched.gitlab import NO_MILESTONE_TAG from grimoire_elk.enriched.utils import REPO_LABELS @@ -178,21 +177,6 @@ def test_refresh_project(self): result = self._test_refresh_project() # ... ? - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - url = "https://gitlab.com/gitlab-org/gitlab-ee --blacklist-ids 1 10 100 --max-retries 100" - arthur_params = {'owner': 'gitlab-org', - 'repository': 'gitlab-ee', - 'blacklist_ids': [1, 10, 100]} - - self.assertDictEqual(arthur_params, GitLabOcean.get_arthur_params_from_url(url)) - - url = "https://gitlab.com/gitlab-org/gitlab-ee --blacklist-ids 1 10 100 --max-retries 100" - perceval_params = ['gitlab-org', 'gitlab-ee', '--blacklist-ids', '1', '10', '100', '--max-retries', '100'] - - self.assertListEqual(perceval_params, GitLabOcean.get_perceval_params_from_url(url)) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_google_hits.py b/tests/test_google_hits.py index 3a6a0231f..2137968b9 100644 --- a/tests/test_google_hits.py +++ b/tests/test_google_hits.py @@ -18,12 +18,10 @@ # Authors: # Valerio Cosentino # -import json import logging import unittest from base import TestBaseBackend -from grimoire_elk.raw.google_hits import GoogleHitsOcean from grimoire_elk.enriched.utils import REPO_LABELS @@ -86,14 +84,6 @@ def test_refresh_project(self): result = self._test_refresh_project() # ... ? - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - with open("data/projects-release.json") as projects_filename: - url = json.load(projects_filename)['grimoire']['google_hits'][0] - arthur_params = 'bitergia grimoirelab' - self.assertEqual(arthur_params, GoogleHitsOcean.get_arthur_params_from_url(url)) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_groupsio.py b/tests/test_groupsio.py index e665f4d15..b29ab1bf8 100644 --- a/tests/test_groupsio.py +++ b/tests/test_groupsio.py @@ -19,12 +19,10 @@ # Alvaro del Castillo # Valerio Cosentino # -import json import logging import unittest from base import TestBaseBackend -from grimoire_elk.raw.groupsio import GroupsioOcean from grimoire_elk.enriched.utils import REPO_LABELS @@ -97,14 +95,6 @@ def test_refresh_project(self): result = self._test_refresh_project() # ... ? - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - with open("data/projects-release.json") as projects_filename: - url = json.load(projects_filename)['grimoire']['groupsio'][0] - arthur_params = {'dirpath': '/tmp', 'group_name': 'updates'} - self.assertDictEqual(arthur_params, GroupsioOcean.get_arthur_params_from_url(url)) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_hyperkitty.py b/tests/test_hyperkitty.py index 14d3ee6bf..e3511fe56 100644 --- a/tests/test_hyperkitty.py +++ b/tests/test_hyperkitty.py @@ -19,13 +19,11 @@ # Alvaro del Castillo # Valerio Cosentino # -import json import logging import unittest from base import TestBaseBackend -from grimoire_elk.raw.hyperkitty import HyperKittyOcean from grimoire_elk.enriched.utils import REPO_LABELS @@ -106,15 +104,6 @@ def test_refresh_project(self): result = self._test_refresh_project() # ... ? - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - with open("data/projects-release.json") as projects_filename: - url = json.load(projects_filename)['grimoire']['hyperkitty'][0] - arthur_params = {'uri': 'https://lists.mailman3.org/archives/list/mailman-users@mailman3.org', - 'url': 'https://lists.mailman3.org/archives/list/mailman-users@mailman3.org'} - self.assertDictEqual(arthur_params, HyperKittyOcean.get_arthur_params_from_url(url)) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_jenkins.py b/tests/test_jenkins.py index 013a3feec..42b8fa335 100644 --- a/tests/test_jenkins.py +++ b/tests/test_jenkins.py @@ -125,16 +125,6 @@ def test_perceval_params(self): ] self.assertListEqual(JenkinsOcean.get_perceval_params_from_url(url), expected_params) - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - url = 'https://build.opnfv.org/ci' - expected_params = { - 'uri': 'https://build.opnfv.org/ci', - 'url': 'https://build.opnfv.org/ci' - } - self.assertDictEqual(JenkinsOcean.get_arthur_params_from_url(url), expected_params) - def test_p2o_params(self): """Test the extraction of p2o params from an URL""" diff --git a/tests/test_jira.py b/tests/test_jira.py index fef98df9c..0d09f7a02 100644 --- a/tests/test_jira.py +++ b/tests/test_jira.py @@ -177,14 +177,6 @@ def test_refresh_project(self): result = self._test_refresh_project() # ... ? - def test_arthur_params(self): - """Test the extraction of arthur params from the projects.json entry""" - - with open("data/projects-release.json") as projects_filename: - url = json.load(projects_filename)['grimoire']['jira'][0] - arthur_params = {'url': 'https://jira.opnfv.org'} - self.assertDictEqual(arthur_params, JiraOcean.get_arthur_params_from_url(url)) - def test_get_p2o_params_from_url(self): """Test the extraction of p2o params from the projects.json entry""" diff --git a/tests/test_kitsune.py b/tests/test_kitsune.py index 42a604375..2fdcf4f3e 100644 --- a/tests/test_kitsune.py +++ b/tests/test_kitsune.py @@ -19,12 +19,10 @@ # Alvaro del Castillo # Valerio Cosentino # -import json import logging import unittest from base import TestBaseBackend -from grimoire_elk.raw.kitsune import KitsuneOcean from grimoire_elk.enriched.utils import REPO_LABELS @@ -91,14 +89,6 @@ def test_refresh_project(self): result = self._test_refresh_project() # ... ? - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - with open("data/projects-release.json") as projects_filename: - url = json.load(projects_filename)['grimoire']['kitsune'][0] - arthur_params = {'uri': '', 'url': ''} - self.assertDictEqual(arthur_params, KitsuneOcean.get_arthur_params_from_url(url)) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_mattermost.py b/tests/test_mattermost.py index 71bd8980b..115427e0a 100644 --- a/tests/test_mattermost.py +++ b/tests/test_mattermost.py @@ -111,16 +111,6 @@ def test_perceval_params(self): ] self.assertListEqual(MattermostOcean.get_perceval_params_from_url(url), expected_params) - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - url = "https://chat.openshift.io 8j366ft5affy3p36987pcugaoa" - expected_params = { - 'channel': '8j366ft5affy3p36987pcugaoa', - 'url': 'https://chat.openshift.io' - } - self.assertDictEqual(MattermostOcean.get_arthur_params_from_url(url), expected_params) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_mbox.py b/tests/test_mbox.py index 88fc39d04..5867ea346 100644 --- a/tests/test_mbox.py +++ b/tests/test_mbox.py @@ -161,16 +161,6 @@ def test_perceval_params(self): ] self.assertListEqual(MBoxOcean.get_perceval_params_from_url(url), expected_params) - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - url = "metrics-grimoire /home/bitergia/.perceval/mbox" - expected_params = { - 'dirpath': '/home/bitergia/.perceval/mbox', - 'uri': 'metrics-grimoire' - } - self.assertDictEqual(MBoxOcean.get_arthur_params_from_url(url), expected_params) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_mediawiki.py b/tests/test_mediawiki.py index cdfc41e0e..34f9cb7cc 100644 --- a/tests/test_mediawiki.py +++ b/tests/test_mediawiki.py @@ -225,15 +225,6 @@ def test_perceval_params(self): ] self.assertListEqual(MediaWikiOcean.get_perceval_params_from_url(url), expected_params) - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - url = "https://wiki.mozilla.org" - expected_params = { - 'url': 'https://wiki.mozilla.org' - } - self.assertDictEqual(MediaWikiOcean.get_arthur_params_from_url(url), expected_params) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_meetup.py b/tests/test_meetup.py index 1c28ccb4a..823d5aaaa 100644 --- a/tests/test_meetup.py +++ b/tests/test_meetup.py @@ -111,15 +111,6 @@ def test_perceval_params(self): ] self.assertListEqual(MeetupOcean.get_perceval_params_from_url(url), expected_params) - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - url = "South-East-Puppet-User-Group" - expected_params = { - 'group': 'South-East-Puppet-User-Group' - } - self.assertDictEqual(MeetupOcean.get_arthur_params_from_url(url), expected_params) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_mozillaclub.py b/tests/test_mozillaclub.py index a5d9e2332..6851317ad 100644 --- a/tests/test_mozillaclub.py +++ b/tests/test_mozillaclub.py @@ -19,12 +19,10 @@ # Alvaro del Castillo # Valerio Cosentino # -import json import logging import unittest from base import TestBaseBackend -from grimoire_elk.raw.mozillaclub import MozillaClubOcean from grimoire_elk.enriched.utils import REPO_LABELS @@ -118,16 +116,6 @@ def test_refresh_project(self): result = self._test_refresh_project() # ... ? - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - with open("data/projects-release.json") as projects_filename: - url = json.load(projects_filename)['grimoire']['mozillaclub'][0] - url_spread = 'https://spreadsheets.google.com/feeds/cells/' - url_spread += '1QHl2bjBhMslyFzR5XXPzMLdzzx7oeSKTbgR5PM8qp64/ohaibtm/public/values?alt=json' - arthur_params = {'uri': url_spread, 'url': url_spread} - self.assertDictEqual(arthur_params, MozillaClubOcean.get_arthur_params_from_url(url)) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_nntp.py b/tests/test_nntp.py index c9857c2a6..be0842a9a 100644 --- a/tests/test_nntp.py +++ b/tests/test_nntp.py @@ -126,16 +126,6 @@ def test_perceval_params(self): ] self.assertListEqual(NNTPOcean.get_perceval_params_from_url(url), expected_params) - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - url = "news.mozilla.org mozilla.dev.project-link" - expected_params = { - 'group': 'mozilla.dev.project-link', - 'host': 'news.mozilla.org' - } - self.assertDictEqual(NNTPOcean.get_arthur_params_from_url(url), expected_params) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_phabricator.py b/tests/test_phabricator.py index e03ed9705..349f48e5c 100644 --- a/tests/test_phabricator.py +++ b/tests/test_phabricator.py @@ -19,12 +19,10 @@ # Alvaro del Castillo # Valerio Cosentino # -import json import logging import unittest from base import TestBaseBackend -from grimoire_elk.raw.phabricator import PhabricatorOcean from grimoire_elk.enriched.utils import REPO_LABELS @@ -91,14 +89,6 @@ def test_refresh_project(self): result = self._test_refresh_project() # ... ? - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - with open("data/projects-release.json") as projects_filename: - url = json.load(projects_filename)['grimoire']['phabricator'][0] - arthur_params = {'uri': 'https://phabricator.wikimedia.org', 'url': 'https://phabricator.wikimedia.org'} - self.assertDictEqual(arthur_params, PhabricatorOcean.get_arthur_params_from_url(url)) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_pipermail.py b/tests/test_pipermail.py index 1d0106dea..96a9ccabe 100644 --- a/tests/test_pipermail.py +++ b/tests/test_pipermail.py @@ -19,12 +19,10 @@ # Alvaro del Castillo # Valerio Cosentino # -import json import logging import unittest from base import TestBaseBackend -from grimoire_elk.raw.pipermail import PipermailOcean from grimoire_elk.enriched.utils import REPO_LABELS @@ -101,14 +99,6 @@ def test_refresh_project(self): result = self._test_refresh_project() # ... ? - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - with open("data/projects-release.json") as projects_filename: - url = json.load(projects_filename)['grimoire']['pipermail'][0] - arthur_params = {'dirpath': '/tmp', 'url': 'https://mail.gnome.org/archives/libart-hackers/'} - self.assertDictEqual(arthur_params, PipermailOcean.get_arthur_params_from_url(url)) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_puppetforge.py b/tests/test_puppetforge.py index 064e463b7..1e4f44c0b 100644 --- a/tests/test_puppetforge.py +++ b/tests/test_puppetforge.py @@ -19,12 +19,10 @@ # Alvaro del Castillo # Valerio Cosentino # -import json import logging import unittest from base import TestBaseBackend -from grimoire_elk.raw.puppetforge import PuppetForgeOcean from grimoire_elk.enriched.utils import REPO_LABELS @@ -91,14 +89,6 @@ def test_refresh_project(self): result = self._test_refresh_project() # ... ? - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - with open("data/projects-release.json") as projects_filename: - url = json.load(projects_filename)['grimoire']['puppetforge'][0] - arthur_params = {'uri': '', 'url': ''} - self.assertDictEqual(arthur_params, PuppetForgeOcean.get_arthur_params_from_url(url)) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_redmine.py b/tests/test_redmine.py index 3e872f3aa..94c3e9674 100644 --- a/tests/test_redmine.py +++ b/tests/test_redmine.py @@ -19,12 +19,10 @@ # Alvaro del Castillo # Valerio Cosentino # -import json import logging import unittest from base import TestBaseBackend -from grimoire_elk.raw.redmine import RedmineOcean from grimoire_elk.enriched.utils import REPO_LABELS @@ -113,14 +111,6 @@ def test_refresh_project(self): result = self._test_refresh_project() # ... ? - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - with open("data/projects-release.json") as projects_filename: - url = json.load(projects_filename)['grimoire']['redmine'][0] - arthur_params = {'uri': 'http://tracker.ceph.com/', 'url': 'http://tracker.ceph.com/'} - self.assertDictEqual(arthur_params, RedmineOcean.get_arthur_params_from_url(url)) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_remo.py b/tests/test_remo.py index 039195ac1..3ef2464ab 100644 --- a/tests/test_remo.py +++ b/tests/test_remo.py @@ -19,12 +19,10 @@ # Alvaro del Castillo # Valerio Cosentino # -import json import logging import unittest from base import TestBaseBackend -from grimoire_elk.raw.remo import ReMoOcean from grimoire_elk.enriched.utils import REPO_LABELS @@ -91,14 +89,6 @@ def test_refresh_project(self): result = self._test_refresh_project() # ... ? - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - with open("data/projects-release.json") as projects_filename: - url = json.load(projects_filename)['grimoire']['remo'][0] - arthur_params = {'uri': 'https://reps.mozilla.org', 'url': 'https://reps.mozilla.org'} - self.assertDictEqual(arthur_params, ReMoOcean.get_arthur_params_from_url(url)) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_rss.py b/tests/test_rss.py index 2d35b8cc8..6b6c113e1 100644 --- a/tests/test_rss.py +++ b/tests/test_rss.py @@ -19,12 +19,10 @@ # Alvaro del Castillo # Valerio Cosentino # -import json import logging import unittest from base import TestBaseBackend -from grimoire_elk.raw.rss import RSSOcean from grimoire_elk.enriched.utils import REPO_LABELS @@ -91,14 +89,6 @@ def test_refresh_project(self): result = self._test_refresh_project() # ... ? - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - with open("data/projects-release.json") as projects_filename: - url = json.load(projects_filename)['grimoire']['rss'][0] - arthur_params = {'uri': 'https://blog.bitergia.com/feed/', 'url': 'https://blog.bitergia.com/feed/'} - self.assertDictEqual(arthur_params, RSSOcean.get_arthur_params_from_url(url)) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_slack.py b/tests/test_slack.py index a901d39ca..a5d55429b 100644 --- a/tests/test_slack.py +++ b/tests/test_slack.py @@ -19,14 +19,12 @@ # Alvaro del Castillo # Valerio Cosentino # -import json import logging import unittest import requests from base import TestBaseBackend from grimoire_elk.enriched.utils import REPO_LABELS -from grimoire_elk.raw.slack import SlackOcean class TestSlack(TestBaseBackend): @@ -133,14 +131,6 @@ def test_refresh_project(self): result = self._test_refresh_project() # ... ? - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - with open("data/projects-release.json") as projects_filename: - url = json.load(projects_filename)['grimoire']['slack'][0] - arthur_params = {'channel': 'C011DUKE8'} - self.assertDictEqual(arthur_params, SlackOcean.get_arthur_params_from_url(url)) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_stackexchange.py b/tests/test_stackexchange.py index 359883865..66ee30982 100644 --- a/tests/test_stackexchange.py +++ b/tests/test_stackexchange.py @@ -101,16 +101,6 @@ def test_perceval_params(self): ] self.assertListEqual(StackExchangeOcean.get_perceval_params_from_url(url), expected_params) - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - url = "https://stackoverflow.com/questions/tagged/ovirt" - expected_params = { - 'site': 'stackoverflow.com', - 'tagged': 'ovirt' - } - self.assertDictEqual(StackExchangeOcean.get_arthur_params_from_url(url), expected_params) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_supybot.py b/tests/test_supybot.py index 47ec0d2eb..e4011ea2a 100644 --- a/tests/test_supybot.py +++ b/tests/test_supybot.py @@ -100,16 +100,6 @@ def test_perceval_params(self): ] self.assertListEqual(SupybotOcean.get_perceval_params_from_url(url), expected_params) - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - url = "openshift /home/bitergia/.perceval/irc/percevalbot/logs/ChannelLogger/freenode/#openshift/" - expected_params = { - 'dirpath': '/home/bitergia/.perceval/irc/percevalbot/logs/ChannelLogger/freenode/#openshift/', - 'uri': 'openshift' - } - self.assertDictEqual(SupybotOcean.get_arthur_params_from_url(url), expected_params) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_telegram.py b/tests/test_telegram.py index 1189ff907..7edec6f07 100644 --- a/tests/test_telegram.py +++ b/tests/test_telegram.py @@ -19,12 +19,10 @@ # Alvaro del Castillo # Valerio Cosentino # -import json import logging import unittest from base import TestBaseBackend -from grimoire_elk.raw.telegram import TelegramOcean from grimoire_elk.enriched.utils import REPO_LABELS @@ -117,14 +115,6 @@ def test_refresh_project(self): result = self._test_refresh_project() # ... ? - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - with open("data/projects-release.json") as projects_filename: - url = json.load(projects_filename)['grimoire']['telegram'][0] - arthur_params = {'bot': 'Mozilla_analytics'} - self.assertDictEqual(arthur_params, TelegramOcean.get_arthur_params_from_url(url)) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/tests/test_twitter.py b/tests/test_twitter.py index dfe8abc36..2a3277b1c 100644 --- a/tests/test_twitter.py +++ b/tests/test_twitter.py @@ -25,7 +25,6 @@ from base import TestBaseBackend from grimoire_elk.enriched.utils import REPO_LABELS -from grimoire_elk.raw.twitter import TwitterOcean class TestTwitter(TestBaseBackend): @@ -91,15 +90,6 @@ def test_refresh_project(self): result = self._test_refresh_project() # ... ? - def test_arthur_params(self): - """Test the extraction of arthur params from an URL""" - - url = "bitergia" - expected_params = { - 'query': 'bitergia' - } - self.assertDictEqual(TwitterOcean.get_arthur_params_from_url(url), expected_params) - if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') diff --git a/utils/p2o.py b/utils/p2o.py index 66ea74adb..49315fcc2 100755 --- a/utils/p2o.py +++ b/utils/p2o.py @@ -57,8 +57,7 @@ if not args.enrich_only: feed_backend(url, clean, args.fetch_cache, args.backend, args.backend_args, - args.index, args.index_enrich, args.project, - args.arthur) + args.index, args.index_enrich, args.project) logging.info("Backend feed completed") studies_args = None