From 1894b2c7d6925bebf350c6ae149225bc01032e6d Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Thu, 21 Sep 2023 14:32:48 +0200 Subject: [PATCH 01/11] Discover bots based on the entry points The bot discovery algorithm was rewritten to use entry point names instead of the file structure. This allow really easy developing of packages with custom bots. Not that the 'group' feature of entry points wasn't used because we require bots to register callable scripts that are later used to start bot's processes. --- .gitignore | 2 +- CHANGELOG.md | 2 + .../mybots/__init__.py | 0 .../mybots/bots/__init__.py | 0 .../mybots/bots/collectors/__init__.py | 0 .../mybots/bots/collectors/custom/__init__.py | 0 .../bots/collectors/custom/collector.py | 16 ++++++ contrib/example-extension-package/setup.py | 38 +++++++++++++ docs/dev/guide.rst | 57 +++++++++++++++++++ intelmq/lib/utils.py | 13 ++--- 10 files changed, 120 insertions(+), 8 deletions(-) create mode 100644 contrib/example-extension-package/mybots/__init__.py create mode 100644 contrib/example-extension-package/mybots/bots/__init__.py create mode 100644 contrib/example-extension-package/mybots/bots/collectors/__init__.py create mode 100644 contrib/example-extension-package/mybots/bots/collectors/custom/__init__.py create mode 100644 contrib/example-extension-package/mybots/bots/collectors/custom/collector.py create mode 100644 contrib/example-extension-package/setup.py diff --git a/.gitignore b/.gitignore index bac7657c5..4b098a8c3 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,7 @@ *.profile .vscode/ .profile -intelmq.egg-info +*.egg-info build dist *.old diff --git a/CHANGELOG.md b/CHANGELOG.md index d4d430389..9de3a25aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ CHANGELOG ### Core - `intelmq.lib.message`: For invalid message keys, add a hint on the failure to the exception: not allowed by configuration or not matching regular expression (PR#2398 by Sebastian Wagner). - `intelmq.lib.exceptions.InvalidKey`: Add optional parameter `additional_text` (PR#2398 by Sebastian Wagner). +- Change the way we discover bots to allow easy extending based on the entry point name. (PR by Kamil Mankowski) ### Development @@ -31,6 +32,7 @@ CHANGELOG ### Documentation - Add a readthedocs configuration file to fix the build fail (PR#2403 by Sebastian Wagner). +- Add a guide of developing extensions packages (PR by Kamil Mankowski) ### Packaging diff --git a/contrib/example-extension-package/mybots/__init__.py b/contrib/example-extension-package/mybots/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/contrib/example-extension-package/mybots/bots/__init__.py b/contrib/example-extension-package/mybots/bots/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/contrib/example-extension-package/mybots/bots/collectors/__init__.py b/contrib/example-extension-package/mybots/bots/collectors/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/contrib/example-extension-package/mybots/bots/collectors/custom/__init__.py b/contrib/example-extension-package/mybots/bots/collectors/custom/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/contrib/example-extension-package/mybots/bots/collectors/custom/collector.py b/contrib/example-extension-package/mybots/bots/collectors/custom/collector.py new file mode 100644 index 000000000..89122e015 --- /dev/null +++ b/contrib/example-extension-package/mybots/bots/collectors/custom/collector.py @@ -0,0 +1,16 @@ +from intelmq.lib.bot import CollectorBot + + +class ExampleAdditionalCollectorBot(CollectorBot): + """ + This is an example bot provided by extension package + """ + + def process(self): + report = self.new_report() + if self.raw: # noqa: Set as parameter + report['raw'] = 'test' + self.send_message(report) + + +BOT = ExampleAdditionalCollectorBot diff --git a/contrib/example-extension-package/setup.py b/contrib/example-extension-package/setup.py new file mode 100644 index 000000000..a97e7bb18 --- /dev/null +++ b/contrib/example-extension-package/setup.py @@ -0,0 +1,38 @@ +from pathlib import Path +from setuptools import find_packages, setup + + +# Instead of the bot-autodiscovery below, you can also just manually declare entrypoints +# (regardless of packaging solution, even in pyproject.toml etc.), e.g.: +# +# intelmq.bots.collectors.custom.collector = mybots.bots.collectors.custom.collector:BOT.run +# +# Important is: +# - entry point has to start with `intelmq.bots.{type}` (type: collectors, experts, parsers, outputs) +# - target has to end with `:BOT.run` +# - entry points have to be in `console_scripts` group + + +BOTS = [] + +base_path = Path(__file__).parent / 'mybots/bots' +botfiles = [botfile for botfile in Path(base_path).glob('**/*.py') if botfile.is_file() and not botfile.name.startswith('_')] +for file in botfiles: + file = Path(str(file).replace(str(base_path), 'intelmq/bots')) + entry_point = '.'.join(file.with_suffix('').parts) + file = Path(str(file).replace('intelmq/bots', 'mybots/bots')) + module = '.'.join(file.with_suffix('').parts) + BOTS.append('{0} = {1}:BOT.run'.format(entry_point, module)) + +setup( + name='intelmq-example-extension', + version='1.0.0', # noqa: F821 + maintainer='Your Name', + maintainer_email='you@example.com', + packages=find_packages(), + license='AGPLv3', + description='This is an example package to demonstrate how ones can extend IntelMQ.', + entry_points={ + 'console_scripts': BOTS + }, +) diff --git a/docs/dev/guide.rst b/docs/dev/guide.rst index 1857f291b..58041cb6d 100644 --- a/docs/dev/guide.rst +++ b/docs/dev/guide.rst @@ -891,6 +891,63 @@ The databases `<` 10 are reserved for the IntelMQ core: * 3: statistics * 4: tests +**************************** +Creating extensions packages +**************************** + +IntelMQ supports adding additional bots using your own independent packages. You can use this to +add a new integration that is special to you, or cannot be integrated for some reason +into the main IntelMQ repository. + +Building an extension package +============================= + +A simple example of the package can be found in ``contrib/example-extension-package``. In order to +bots to work with IntelMQ, you need to ensure that + + - your bot's module exposes a ``BOT`` object of the class inherited from ``intelmq.lib.bot.Bot`` + or its subclasses, + - your package registers an `entry point `_ + in the ``console_scripts`` group with a name starting with ``intelmq.bots.`` followed by + the name of the group (collectors, experts, outputs, parsers), and then your original name. + The entry point must point to the ``BOT.run`` method, + - the module in which the bot resides must be importable by IntelMQ (e.g. installed in the same + virtualv). + +Apart from these requirements, your package can use any of the usual package features. We strongly +recommend following the same principles and main guidelines as the official bots. This will ensure +experience when using official and additional bots. + +Naming convention +================= + +Building your own extensions gives you a lot of freedom, but it's important to know that if your +bot's entry point uses the same name as another bot, it may not be possible to use it, or to +determine which one is being used. For this reason, we recommend that you start the name of your +bot with an with an organization identifier and then the bot name. + +For example, if I create a collector bot for feed source ``Special`` and run it on behalf of the +organization ``Awesome``, the suggested entry point might be ``intelmq.bots.collectors.awesome.special``. +Note that the structure of your package doesn't matter, as long as it can be imported properly. + +For example, I could create a package called ``awesome-bots`` with the following file structure + +.. code-block:: text + + awesome_bots + ├── pyproject.toml + └── awesome_bots + ├── __init__.py + └── special.py + +The `pyproject.toml `_ +file would then have the following section: + +.. code-block:: ini + + [project.scripts] + intelmq.bots.collectors.awesome.special = "awesome_bots.special:BOT.run" + ************* Documentation ************* diff --git a/intelmq/lib/utils.py b/intelmq/lib/utils.py index dd926cfac..a4710668b 100644 --- a/intelmq/lib/utils.py +++ b/intelmq/lib/utils.py @@ -47,6 +47,7 @@ from ruamel.yaml import YAML from ruamel.yaml.scanner import ScannerError from termstyle import red +from importlib.metadata import entry_points import intelmq from intelmq import RUNTIME_CONF_FILE @@ -860,13 +861,11 @@ def list_all_bots() -> dict: from intelmq.lib.bot import Bot # noqa: prevents circular import bot_parameters = dir(Bot) - base_path = resource_filename('intelmq', 'bots') - - botfiles = [botfile for botfile in pathlib.Path(base_path).glob('**/*.py') if botfile.is_file() and botfile.name != '__init__.py'] - for file in botfiles: - file = Path(file.as_posix().replace(base_path, 'intelmq/bots')) + bot_entrypoints = filter(lambda entry: entry.name.startswith("intelmq.bots."), entry_points(group="console_scripts")) + for bot in bot_entrypoints: try: - mod = importlib.import_module('.'.join(file.with_suffix('').parts)) + module_name = bot.value.replace(":BOT.run", '') + mod = importlib.import_module(module_name) except SyntaxError: # Skip invalid bots continue @@ -884,7 +883,7 @@ def list_all_bots() -> dict: for bot_type in ['CollectorBot', 'ParserBot', 'ExpertBot', 'OutputBot', 'Bot']: name = name.replace(bot_type, '') - bots[file.parts[2].capitalize()[:-1]][name] = { + bots[module_name.split('.')[2].capitalize()[:-1]][name] = { "module": mod.__name__, "description": "Missing description" if not getattr(mod.BOT, '__doc__', None) else textwrap.dedent(mod.BOT.__doc__).strip(), "parameters": keys, From 55e916614f9066e12ffc6b973695d97d028ca9fd Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Thu, 21 Sep 2023 14:42:29 +0200 Subject: [PATCH 02/11] Fix licenses and dependencies --- .../mybots/bots/collectors/custom/collector.py | 5 +++++ contrib/example-extension-package/setup.py | 6 ++++++ intelmq/lib/utils.py | 9 ++++++--- setup.py | 1 + 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/contrib/example-extension-package/mybots/bots/collectors/custom/collector.py b/contrib/example-extension-package/mybots/bots/collectors/custom/collector.py index 89122e015..33ec5754a 100644 --- a/contrib/example-extension-package/mybots/bots/collectors/custom/collector.py +++ b/contrib/example-extension-package/mybots/bots/collectors/custom/collector.py @@ -1,3 +1,8 @@ +""" +SPDX-FileCopyrightText: 2023 CERT.at GmbH +SPDX-License-Identifier: AGPL-3.0-or-later +""" + from intelmq.lib.bot import CollectorBot diff --git a/contrib/example-extension-package/setup.py b/contrib/example-extension-package/setup.py index a97e7bb18..5bf1e75f5 100644 --- a/contrib/example-extension-package/setup.py +++ b/contrib/example-extension-package/setup.py @@ -1,3 +1,9 @@ +"""Example IntelMQ extension package + +SPDX-FileCopyrightText: 2023 CERT.at GmbH +SPDX-License-Identifier: AGPL-3.0-or-later +""" + from pathlib import Path from setuptools import find_packages, setup diff --git a/intelmq/lib/utils.py b/intelmq/lib/utils.py index a4710668b..77df16717 100644 --- a/intelmq/lib/utils.py +++ b/intelmq/lib/utils.py @@ -34,7 +34,6 @@ import textwrap import traceback import zipfile -from pathlib import Path from typing import (Any, Callable, Dict, Generator, Iterator, Optional, Sequence, Union) @@ -43,16 +42,20 @@ import dns.version import requests from dateutil.relativedelta import relativedelta -from pkg_resources import resource_filename from ruamel.yaml import YAML from ruamel.yaml.scanner import ScannerError from termstyle import red -from importlib.metadata import entry_points import intelmq from intelmq import RUNTIME_CONF_FILE from intelmq.lib.exceptions import DecodingError +try: + from importlib.metadata import entry_points +except ImportError: + from importlib_metadata import entry_points + + __all__ = ['base64_decode', 'base64_encode', 'decode', 'encode', 'load_configuration', 'load_parameters', 'log', 'parse_logline', 'reverse_readline', 'error_message_from_exc', 'parse_relative', diff --git a/setup.py b/setup.py index b19b2f8f5..7b797416e 100644 --- a/setup.py +++ b/setup.py @@ -17,6 +17,7 @@ 'redis>=2.10', 'requests>=2.2.0', 'ruamel.yaml', + 'importlib-metadata; python_version < "3.8"' ] TESTS_REQUIRES = [ From 9d54c5c64ad4206abfd2bfa2ba8fb96e5156c552 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Thu, 21 Sep 2023 14:57:45 +0200 Subject: [PATCH 03/11] Fix backward compatibility --- intelmq/lib/utils.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/intelmq/lib/utils.py b/intelmq/lib/utils.py index 77df16717..31333ad0c 100644 --- a/intelmq/lib/utils.py +++ b/intelmq/lib/utils.py @@ -34,6 +34,7 @@ import textwrap import traceback import zipfile +from sys import version_info from typing import (Any, Callable, Dict, Generator, Iterator, Optional, Sequence, Union) @@ -843,6 +844,13 @@ def file_name_from_response(response: requests.Response) -> str: return file_name +def _get_console_entry_points(): + # Select interface was introduced in Python 3.10 + if version_info < (3, 10): + return entry_points().get("console_scripts", []) + return entry_points(group="console_scripts") + + def list_all_bots() -> dict: """ Compile a dictionary with all bots and their parameters. @@ -864,7 +872,7 @@ def list_all_bots() -> dict: from intelmq.lib.bot import Bot # noqa: prevents circular import bot_parameters = dir(Bot) - bot_entrypoints = filter(lambda entry: entry.name.startswith("intelmq.bots."), entry_points(group="console_scripts")) + bot_entrypoints = filter(lambda entry: entry.name.startswith("intelmq.bots."), _get_console_entry_points()) for bot in bot_entrypoints: try: module_name = bot.value.replace(":BOT.run", '') From c635e0d3da683d13ae04594efae2ac714001b790 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Thu, 21 Sep 2023 15:05:35 +0200 Subject: [PATCH 04/11] Fix calling object --- intelmq/lib/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intelmq/lib/utils.py b/intelmq/lib/utils.py index 31333ad0c..cb46a3f62 100644 --- a/intelmq/lib/utils.py +++ b/intelmq/lib/utils.py @@ -847,7 +847,7 @@ def file_name_from_response(response: requests.Response) -> str: def _get_console_entry_points(): # Select interface was introduced in Python 3.10 if version_info < (3, 10): - return entry_points().get("console_scripts", []) + return entry_points()["console_scripts"] return entry_points(group="console_scripts") From 548d15a6f254ec8e9cc3a7fb6f27861bb0f1e778 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Thu, 21 Sep 2023 15:29:00 +0200 Subject: [PATCH 05/11] Fix compatibility and note testing installation --- docs/dev/guide.rst | 3 +++ intelmq/lib/utils.py | 9 +++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/dev/guide.rst b/docs/dev/guide.rst index 58041cb6d..04b679b66 100644 --- a/docs/dev/guide.rst +++ b/docs/dev/guide.rst @@ -948,6 +948,9 @@ file would then have the following section: [project.scripts] intelmq.bots.collectors.awesome.special = "awesome_bots.special:BOT.run" +Once you installed your package, you can run ``intelmqctl list bots`` to check if your bot was +properly registered. + ************* Documentation ************* diff --git a/intelmq/lib/utils.py b/intelmq/lib/utils.py index cb46a3f62..162ae89eb 100644 --- a/intelmq/lib/utils.py +++ b/intelmq/lib/utils.py @@ -845,10 +845,11 @@ def file_name_from_response(response: requests.Response) -> str: def _get_console_entry_points(): - # Select interface was introduced in Python 3.10 - if version_info < (3, 10): - return entry_points()["console_scripts"] - return entry_points(group="console_scripts") + # Select interface was introduced in Python 3.10 and newer importlib_metadata + entries = entry_points() + if hasattr(entries, "select"): + return entries.select(group="console_scripts") + return entries.get("console_scripts", []) # it's a dict def list_all_bots() -> dict: From 959ef04abeb967d3a7d13c0233bca8cd5e2dc0af Mon Sep 17 00:00:00 2001 From: kamil-certat <117654481+kamil-certat@users.noreply.github.com> Date: Wed, 27 Sep 2023 10:08:14 +0200 Subject: [PATCH 06/11] Apply suggestions from code review Co-authored-by: Sebastian --- CHANGELOG.md | 4 ++-- .../mybots/bots/collectors/custom/collector.py | 2 +- contrib/example-extension-package/setup.py | 2 +- docs/dev/guide.rst | 12 ++++++------ 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9de3a25aa..e9eebdb34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ CHANGELOG ### Core - `intelmq.lib.message`: For invalid message keys, add a hint on the failure to the exception: not allowed by configuration or not matching regular expression (PR#2398 by Sebastian Wagner). - `intelmq.lib.exceptions.InvalidKey`: Add optional parameter `additional_text` (PR#2398 by Sebastian Wagner). -- Change the way we discover bots to allow easy extending based on the entry point name. (PR by Kamil Mankowski) +- Change the way we discover bots to allow easy extending based on the entry point name. (PR#2413 by Kamil Mankowski) ### Development @@ -32,7 +32,7 @@ CHANGELOG ### Documentation - Add a readthedocs configuration file to fix the build fail (PR#2403 by Sebastian Wagner). -- Add a guide of developing extensions packages (PR by Kamil Mankowski) +- Add a guide of developing extensions packages (PR#2413 by Kamil Mankowski) ### Packaging diff --git a/contrib/example-extension-package/mybots/bots/collectors/custom/collector.py b/contrib/example-extension-package/mybots/bots/collectors/custom/collector.py index 33ec5754a..f015786f8 100644 --- a/contrib/example-extension-package/mybots/bots/collectors/custom/collector.py +++ b/contrib/example-extension-package/mybots/bots/collectors/custom/collector.py @@ -8,7 +8,7 @@ class ExampleAdditionalCollectorBot(CollectorBot): """ - This is an example bot provided by extension package + This is an example bot provided by an extension package """ def process(self): diff --git a/contrib/example-extension-package/setup.py b/contrib/example-extension-package/setup.py index 5bf1e75f5..3d6774693 100644 --- a/contrib/example-extension-package/setup.py +++ b/contrib/example-extension-package/setup.py @@ -11,7 +11,7 @@ # Instead of the bot-autodiscovery below, you can also just manually declare entrypoints # (regardless of packaging solution, even in pyproject.toml etc.), e.g.: # -# intelmq.bots.collectors.custom.collector = mybots.bots.collectors.custom.collector:BOT.run +# 'intelmq.bots.collectors.custom.collector = mybots.bots.collectors.custom.collector:BOT.run' # # Important is: # - entry point has to start with `intelmq.bots.{type}` (type: collectors, experts, parsers, outputs) diff --git a/docs/dev/guide.rst b/docs/dev/guide.rst index 04b679b66..05a86886d 100644 --- a/docs/dev/guide.rst +++ b/docs/dev/guide.rst @@ -896,14 +896,14 @@ Creating extensions packages **************************** IntelMQ supports adding additional bots using your own independent packages. You can use this to -add a new integration that is special to you, or cannot be integrated for some reason -into the main IntelMQ repository. +add a new integration that is special to you, or cannot be integrated +into the main IntelMQ repository for some reason. Building an extension package ============================= -A simple example of the package can be found in ``contrib/example-extension-package``. In order to -bots to work with IntelMQ, you need to ensure that +A simple example of the package can be found in ``contrib/example-extension-package``. To make your custom +bots work with IntelMQ, you need to ensure that - your bot's module exposes a ``BOT`` object of the class inherited from ``intelmq.lib.bot.Bot`` or its subclasses, @@ -912,7 +912,7 @@ bots to work with IntelMQ, you need to ensure that the name of the group (collectors, experts, outputs, parsers), and then your original name. The entry point must point to the ``BOT.run`` method, - the module in which the bot resides must be importable by IntelMQ (e.g. installed in the same - virtualv). + virtualenv, if you use them). Apart from these requirements, your package can use any of the usual package features. We strongly recommend following the same principles and main guidelines as the official bots. This will ensure @@ -948,7 +948,7 @@ file would then have the following section: [project.scripts] intelmq.bots.collectors.awesome.special = "awesome_bots.special:BOT.run" -Once you installed your package, you can run ``intelmqctl list bots`` to check if your bot was +Once you have installed your package, you can run ``intelmqctl list bots`` to check if your bot was properly registered. ************* From 5e313a6a8f8ba86eb74ae867c11dc4d53048ecc0 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Wed, 27 Sep 2023 16:21:07 +0200 Subject: [PATCH 07/11] Test importing bots. Fix debian/control and guide --- .../bots/collectors/custom/collector.py | 5 +++- debian/control | 1 + docs/dev/guide.rst | 2 +- docs/user/installation.rst | 4 +++ intelmq/tests/lib/test_utils.py | 28 +++++++++++++++++++ 5 files changed, 38 insertions(+), 2 deletions(-) diff --git a/contrib/example-extension-package/mybots/bots/collectors/custom/collector.py b/contrib/example-extension-package/mybots/bots/collectors/custom/collector.py index f015786f8..6a53fa4b7 100644 --- a/contrib/example-extension-package/mybots/bots/collectors/custom/collector.py +++ b/contrib/example-extension-package/mybots/bots/collectors/custom/collector.py @@ -3,6 +3,9 @@ SPDX-License-Identifier: AGPL-3.0-or-later """ +# Use your package as usual +from mybots.lib import common + from intelmq.lib.bot import CollectorBot @@ -14,7 +17,7 @@ class ExampleAdditionalCollectorBot(CollectorBot): def process(self): report = self.new_report() if self.raw: # noqa: Set as parameter - report['raw'] = 'test' + report['raw'] = common.return_value('example') self.send_message(report) diff --git a/debian/control b/debian/control index ea1520dda..818178d14 100644 --- a/debian/control +++ b/debian/control @@ -41,6 +41,7 @@ Depends: bash-completion, python3-ruamel.yaml, python3-termstyle (>= 0.1.10), python3-tz, + python3-importlib-metadata, redis-server, systemd, ${misc:Depends}, diff --git a/docs/dev/guide.rst b/docs/dev/guide.rst index 05a86886d..cc9924e35 100644 --- a/docs/dev/guide.rst +++ b/docs/dev/guide.rst @@ -916,7 +916,7 @@ bots work with IntelMQ, you need to ensure that Apart from these requirements, your package can use any of the usual package features. We strongly recommend following the same principles and main guidelines as the official bots. This will ensure -experience when using official and additional bots. +the same experience when using official and additional bots. Naming convention ================= diff --git a/docs/user/installation.rst b/docs/user/installation.rst index 0c026cb78..97215fdb9 100644 --- a/docs/user/installation.rst +++ b/docs/user/installation.rst @@ -45,6 +45,10 @@ Native deb/rpm packages These are the operating systems which are currently supported by packages: +* **Debian 10** Buster + + * Enable the backport repository by the line ``deb-src http://deb.debian.org/debian buster-backports main contrib non-free`` to the file ``/etc/apt/sources.list`` first. + * **Debian 11** Bullseye * **openSUSE Tumbleweed** * **Ubuntu 20.04** Focal Fossa diff --git a/intelmq/tests/lib/test_utils.py b/intelmq/tests/lib/test_utils.py index 730517c8f..f0fac0b8f 100644 --- a/intelmq/tests/lib/test_utils.py +++ b/intelmq/tests/lib/test_utils.py @@ -31,6 +31,12 @@ from intelmq.lib.test import skip_internet from intelmq.tests.test_conf import CerberusTests +try: + from importlib.metadata import EntryPoint +except ImportError: + from importlib_metadata import EntryPoint + + LINES = {'spare': ['Lorem', 'ipsum', 'dolor'], 'short': ['{}: Lorem', '{}: ipsum', '{}: dolor'], @@ -318,6 +324,28 @@ def _mock_importing(module): bot_count = sum([len(val) for val in bots.values()]) self.assertEqual(1, bot_count) + def test_list_all_bots_filters_entrypoints(self): + entries = [ + EntryPoint("intelmq.bots.collector.api.collector_api", + "intelmq.bots.collector.api.collector_api:BOT.run", group="console_scripts"), + EntryPoint("intelmq.bots.collector.awesome.my_bot", + "awesome.extension.package.collector:BOT.run", group="console_scripts"), + EntryPoint("not.a.bot", "not.a.bot:run", group="console_scripts") + ] + + with unittest.mock.patch.object(utils, "_get_console_entry_points", return_value=entries): + with unittest.mock.patch.object(utils.importlib, "import_module") as import_mock: + import_mock.side_effect = SyntaxError() # stop processing after import try + utils.list_all_bots() + + import_mock.assert_has_calls( + [ + unittest.mock.call("intelmq.bots.collector.api.collector_api"), + unittest.mock.call("awesome.extension.package.collector"), + ] + ) + self.assertEqual(2, import_mock.call_count) + def test_get_bots_settings(self): with unittest.mock.patch.object(utils, "get_runtime", new_get_runtime): runtime = utils.get_bots_settings() From 03f63705c1478b69f334c53ca47e6ae82219d37b Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Thu, 28 Sep 2023 09:27:00 +0200 Subject: [PATCH 08/11] Add missing example files --- contrib/example-extension-package/mybots/lib/__init__.py | 0 contrib/example-extension-package/mybots/lib/common.py | 9 +++++++++ 2 files changed, 9 insertions(+) create mode 100644 contrib/example-extension-package/mybots/lib/__init__.py create mode 100644 contrib/example-extension-package/mybots/lib/common.py diff --git a/contrib/example-extension-package/mybots/lib/__init__.py b/contrib/example-extension-package/mybots/lib/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/contrib/example-extension-package/mybots/lib/common.py b/contrib/example-extension-package/mybots/lib/common.py new file mode 100644 index 000000000..5b03048b8 --- /dev/null +++ b/contrib/example-extension-package/mybots/lib/common.py @@ -0,0 +1,9 @@ +""" + +SPDX-FileCopyrightText: 2023 CERT.at GmbH +SPDX-License-Identifier: AGPL-3.0-or-later +""" + + +def return_value(value): + return value From d5209879f9763116fb617dce0129be68e8e32262 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Thu, 28 Sep 2023 09:33:03 +0200 Subject: [PATCH 09/11] Remove unnecessary deb packages pools --- docs/user/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/installation.rst b/docs/user/installation.rst index 97215fdb9..cfa0c0573 100644 --- a/docs/user/installation.rst +++ b/docs/user/installation.rst @@ -47,7 +47,7 @@ These are the operating systems which are currently supported by packages: * **Debian 10** Buster - * Enable the backport repository by the line ``deb-src http://deb.debian.org/debian buster-backports main contrib non-free`` to the file ``/etc/apt/sources.list`` first. + * Enable the backport repository by the line ``deb http://deb.debian.org/debian buster-backports main`` to the file ``/etc/apt/sources.list`` first. * **Debian 11** Bullseye * **openSUSE Tumbleweed** From fa8f6be558f1fdf406fc626be413e3ebee3dc8f0 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Thu, 28 Sep 2023 14:19:36 +0200 Subject: [PATCH 10/11] Fix usages of importing bot's module --- intelmq/bin/intelmqctl.py | 2 +- intelmq/lib/bot_debugger.py | 2 +- intelmq/lib/utils.py | 13 +++++++++++++ intelmq/tests/bin/test_intelmqctl.py | 12 ++++++++++++ intelmq/tests/lib/test_utils.py | 12 ++++++++---- 5 files changed, 35 insertions(+), 6 deletions(-) diff --git a/intelmq/bin/intelmqctl.py b/intelmq/bin/intelmqctl.py index 380b27fa1..51301b1d8 100644 --- a/intelmq/bin/intelmqctl.py +++ b/intelmq/bin/intelmqctl.py @@ -931,7 +931,7 @@ def check(self, no_connections=False, check_executables=True): if bot_id != 'global': # importable module try: - bot_module = importlib.import_module(bot_config['module']) + bot_module = importlib.import_module(utils.get_bot_module_name(bot_config['module'])) except ImportError as exc: check_logger.error('Incomplete installation: Bot %r not importable: %r.', bot_id, exc) retval = 1 diff --git a/intelmq/lib/bot_debugger.py b/intelmq/lib/bot_debugger.py index 4853bebf4..3694a8ec0 100644 --- a/intelmq/lib/bot_debugger.py +++ b/intelmq/lib/bot_debugger.py @@ -50,7 +50,7 @@ def __init__(self, runtime_configuration, bot_id, run_subcommand=None, console_t self.dryrun = dryrun self.msg = msg self.show = show - module = import_module(self.runtime_configuration['module']) + module = import_module(utils.get_bot_module_name(self.runtime_configuration['module'])) if loglevel: self.leverageLogger(loglevel) diff --git a/intelmq/lib/utils.py b/intelmq/lib/utils.py index 162ae89eb..78e6ed7ff 100644 --- a/intelmq/lib/utils.py +++ b/intelmq/lib/utils.py @@ -852,6 +852,19 @@ def _get_console_entry_points(): return entries.get("console_scripts", []) # it's a dict +def get_bot_module_name(bot_name: str) -> str: + entries = entry_points() + if hasattr(entries, "select"): + entries = entries.select(name=bot_name, group="console_scripts") + else: + entries = [entry for entry in entries.get("console_scripts", []) if entry.name == bot_name] + + if not entries: + return None + else: + return entries[0].value.replace(":BOT.run", '') + + def list_all_bots() -> dict: """ Compile a dictionary with all bots and their parameters. diff --git a/intelmq/tests/bin/test_intelmqctl.py b/intelmq/tests/bin/test_intelmqctl.py index f0a594c0e..88ef46a71 100644 --- a/intelmq/tests/bin/test_intelmqctl.py +++ b/intelmq/tests/bin/test_intelmqctl.py @@ -125,6 +125,18 @@ def test_check_handles_syntaxerror_when_importing_bots(self): self.assertIsNotNone( next(filter(lambda l: "SyntaxError in bot 'test-bot'" in l, captured.output))) + @skip_installation() + @mock.patch.object(utils, "get_bot_module_name", mock.Mock(return_value="mocked-module")) + def test_check_imports_real_bot_module(self): + self._load_default_harmonization() + self._extend_config(self.tmp_runtime, self.BOT_CONFIG) + + # raise SyntaxError to stop checking after import + with mock.patch.object(ctl.importlib, "import_module", mock.Mock(side_effect=SyntaxError)) as import_mock: + self.intelmqctl.check(no_connections=True, check_executables=False) + + import_mock.assert_called_once_with("mocked-module") + if __name__ == '__main__': # pragma: nocover unittest.main() diff --git a/intelmq/tests/lib/test_utils.py b/intelmq/tests/lib/test_utils.py index f0fac0b8f..48b54032e 100644 --- a/intelmq/tests/lib/test_utils.py +++ b/intelmq/tests/lib/test_utils.py @@ -346,6 +346,10 @@ def test_list_all_bots_filters_entrypoints(self): ) self.assertEqual(2, import_mock.call_count) + def test_get_bot_module_name_builtin_bot(self): + found_name = utils.get_bot_module_name("intelmq.bots.collectors.api.collector_api") + self.assertEqual("intelmq.bots.collectors.api.collector_api", found_name) + def test_get_bots_settings(self): with unittest.mock.patch.object(utils, "get_runtime", new_get_runtime): runtime = utils.get_bots_settings() @@ -381,14 +385,14 @@ def test_load_configuration_yaml(self): filename = os.path.join(os.path.dirname(__file__), '../assets/example.yaml') self.assertEqual(utils.load_configuration(filename), { - 'some_string': 'Hello World!', - 'other_string': 'with a : in it', + 'some_string': 'Hello World!', + 'other_string': 'with a : in it', 'now more': ['values', 'in', 'a', 'list'], 'types': -4, 'other': True, 'final': 0.5, - } - ) + } + ) def test_load_configuration_yaml_invalid(self): """ Test load_configuration with an invalid YAML file """ From 41817011ff57d9dd284c094250bb2bd397428526 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Thu, 28 Sep 2023 14:40:25 +0200 Subject: [PATCH 11/11] Fix backward compatibility --- intelmq/lib/utils.py | 2 +- intelmq/tests/lib/test_utils.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/intelmq/lib/utils.py b/intelmq/lib/utils.py index 78e6ed7ff..5701db1af 100644 --- a/intelmq/lib/utils.py +++ b/intelmq/lib/utils.py @@ -855,7 +855,7 @@ def _get_console_entry_points(): def get_bot_module_name(bot_name: str) -> str: entries = entry_points() if hasattr(entries, "select"): - entries = entries.select(name=bot_name, group="console_scripts") + entries = tuple(entries.select(name=bot_name, group="console_scripts")) else: entries = [entry for entry in entries.get("console_scripts", []) if entry.name == bot_name] diff --git a/intelmq/tests/lib/test_utils.py b/intelmq/tests/lib/test_utils.py index 48b54032e..b99a50138 100644 --- a/intelmq/tests/lib/test_utils.py +++ b/intelmq/tests/lib/test_utils.py @@ -350,6 +350,8 @@ def test_get_bot_module_name_builtin_bot(self): found_name = utils.get_bot_module_name("intelmq.bots.collectors.api.collector_api") self.assertEqual("intelmq.bots.collectors.api.collector_api", found_name) + self.assertIsNone(utils.get_bot_module_name("intelmq.not-existing-bot")) + def test_get_bots_settings(self): with unittest.mock.patch.object(utils, "get_runtime", new_get_runtime): runtime = utils.get_bots_settings()