Merge branch 'develop' into shadowserver-dynamic-config

.gitignore

@@ -11,7 +11,7 @@
 *.profile
 .vscode/
 .profile
-intelmq.egg-info
+*.egg-info
 build
 dist
 *.old

CHANGELOG.md

@@ -23,12 +23,14 @@
 ### Core
 - `intelmq.lib.message`: For invalid message keys, add a hint on the failure to the exception: not allowed by configuration or not matching regular expression (PR#2398 by Sebastian Wagner).
 - `intelmq.lib.exceptions.InvalidKey`: Add optional parameter `additional_text` (PR#2398 by Sebastian Wagner).
+- Change the way we discover bots to allow easy extending based on the entry point name. (PR#2413 by Kamil Mankowski)
 - `intelmq.lib.mixins`: Add a new class, `StompMixin` (defined in a new submodule: `stomp`),
   which provides certain common STOMP-bot-specific operations, factored out from
   `intelmq.bots.collectors.stomp.collector` and `intelmq.bots.outputs.stomp.output`
   (PR#2408 by Jan Kaliszewski).
 
 ### Development
+- Makefile: Add codespell and test commands (PR#2425 by Sebastian Wagner).
 
 ### Data Format
 
@@ -68,11 +70,13 @@
 
 ### Documentation
 - Add a readthedocs configuration file to fix the build fail (PR#2403 by Sebastian Wagner).
+- Add a guide of developing extensions packages (PR#2413 by Kamil Mankowski)
 - Update/fix/improve the stuff related to the STOMP bots and integration with the *n6*'s
   Stream API (PR#2408 by Jan Kaliszewski).
 - Complete documentation overhaul. Change to markdown format. Uses the mkdocs-material (PR#2419 by Filip Pokorný).
 
 ### Packaging
+- Add `pendulum` to suggested packages, as it is required for the sieve bot (PR#2424 by Sebastian Wagner).
 
 ### Tests
 

Makefile

@@ -12,4 +12,10 @@ docs: mkdocs.yml docs/* intelmq/etc/feeds.yaml intelmq/etc/harmonization.conf in
 	mkdocs build
 
 clean:
-	rm -rf docs_build .mypy_cache .coverage .pytest_cache dist
+	rm -rf docs_build .mypy_cache .coverage .pytest_cache dist
+
+codespell:
+	codespell -x .github/workflows/codespell.excludelines
+
+test:
+	pytest --no-cov -v intelmq/tests/ && echo "Success!"

contrib/example-extension-package/mybots/bots/collectors/custom/collector.py

@@ -0,0 +1,24 @@
+"""
+SPDX-FileCopyrightText: 2023 CERT.at GmbH <https://cert.at/>
+SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+
+# Use your package as usual
+from mybots.lib import common
+
+from intelmq.lib.bot import CollectorBot
+
+
+class ExampleAdditionalCollectorBot(CollectorBot):
+    """
+    This is an example bot provided by an extension package
+    """
+
+    def process(self):
+        report = self.new_report()
+        if self.raw:  # noqa: Set as parameter
+            report['raw'] = common.return_value('example')
+        self.send_message(report)
+
+
+BOT = ExampleAdditionalCollectorBot

contrib/example-extension-package/mybots/lib/common.py

@@ -0,0 +1,9 @@
+"""
+
+SPDX-FileCopyrightText: 2023 CERT.at GmbH <https://cert.at/>
+SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+
+
+def return_value(value):
+    return value

contrib/example-extension-package/setup.py

@@ -0,0 +1,44 @@
+"""Example IntelMQ extension package
+
+SPDX-FileCopyrightText: 2023 CERT.at GmbH <https://cert.at/>
+SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+
+from pathlib import Path
+from setuptools import find_packages, setup
+
+
+# Instead of the bot-autodiscovery below, you can also just manually declare entrypoints
+# (regardless of packaging solution, even in pyproject.toml etc.), e.g.:
+#
+# 'intelmq.bots.collectors.custom.collector = mybots.bots.collectors.custom.collector:BOT.run'
+#
+# Important is:
+#  - entry point has to start with `intelmq.bots.{type}` (type: collectors, experts, parsers, outputs)
+#  - target has to end with `:BOT.run`
+#  - entry points have to be in `console_scripts` group
+
+
+BOTS = []
+
+base_path = Path(__file__).parent / 'mybots/bots'
+botfiles = [botfile for botfile in Path(base_path).glob('**/*.py') if botfile.is_file() and not botfile.name.startswith('_')]
+for file in botfiles:
+    file = Path(str(file).replace(str(base_path), 'intelmq/bots'))
+    entry_point = '.'.join(file.with_suffix('').parts)
+    file = Path(str(file).replace('intelmq/bots', 'mybots/bots'))
+    module = '.'.join(file.with_suffix('').parts)
+    BOTS.append('{0} = {1}:BOT.run'.format(entry_point, module))
+
+setup(
+    name='intelmq-example-extension',
+    version='1.0.0',  # noqa: F821
+    maintainer='Your Name',
+    maintainer_email='[email protected]',
+    packages=find_packages(),
+    license='AGPLv3',
+    description='This is an example package to demonstrate how ones can extend IntelMQ.',
+    entry_points={
+        'console_scripts': BOTS
+    },
+)

debian/control

@@ -41,6 +41,7 @@ Depends: bash-completion,
          python3-ruamel.yaml,
          python3-termstyle (>= 0.1.10),
          python3-tz,
+         python3-importlib-metadata,
          redis-server,
          systemd,
          ${misc:Depends},
@@ -52,7 +53,8 @@ Suggests: python3-geoip2 (>= 2.2.0),
           python3-pyasn (>= 1.5.0),
           python3-pymongo (>= 2.7.1),
           python3-sleekxmpp (>= 1.3.1),
-          python3-stomp.py (>= 4.1.9)
+          python3-stomp.py (>= 4.1.9),
+          python3-pendulum
 Description: Solution for IT security teams for collecting and processing security feeds
  IntelMQ is a solution for IT security teams (CERTs, CSIRTs, abuse
  departments,...) for collecting and processing security feeds (such as log

docs/dev/extensions-packages.md

@@ -0,0 +1,60 @@
+<!-- comment
+   SPDX-FileCopyrightText: 2023 CERT.at GmbH
+   SPDX-License-Identifier: AGPL-3.0-or-later
+-->
+
+# Creating extensions packages
+
+IntelMQ supports adding additional bots using your own independent packages. You can use this to
+add a new integration that is special to you, or cannot be integrated
+into the main IntelMQ repository for some reason.
+
+## Building an extension package
+
+A simple example of the package can be found in ``contrib/example-extension-package``. To make your custom
+bots work with IntelMQ, you need to ensure that
+
+ - your bot's module exposes a ``BOT`` object of the class inherited from ``intelmq.lib.bot.Bot``
+   or its subclasses,
+ - your package registers an [entry point](https://packaging.python.org/en/latest/specifications/entry-points/)
+   in the ``console_scripts`` group with a name starting with ``intelmq.bots.`` followed by
+   the name of the group (collectors, experts, outputs, parsers), and then your original name.
+   The entry point must point to the ``BOT.run`` method,
+ - the module in which the bot resides must be importable by IntelMQ (e.g. installed in the same
+   virtualenv, if you use them).
+
+Apart from these requirements, your package can use any of the usual package features. We strongly
+recommend following the same principles and main guidelines as the official bots. This will ensure
+the same experience when using official and additional bots.
+
+## Naming convention
+
+Building your own extensions gives you a lot of freedom, but it's important to know that if your
+bot's entry point uses the same name as another bot, it may not be possible to use it, or to
+determine which one is being used. For this reason, we recommend that you start the name of your
+bot with an with an organization identifier and then the bot name.
+
+For example, if I create a collector bot for feed source ``Special`` and run it on behalf of the
+organization ``Awesome``, the suggested entry point might be ``intelmq.bots.collectors.awesome.special``.
+Note that the structure of your package doesn't matter, as long as it can be imported properly.
+
+For example, I could create a package called ``awesome-bots`` with the following file structure
+
+```text
+   awesome_bots
+   ├── pyproject.toml
+   └── awesome_bots
+        ├── __init__.py
+        └── special.py
+```
+
+The [pyproject.toml](https://packaging.python.org/en/latest/specifications/declaring-project-metadata/#entry-points)
+file would then have the following section:
+
+```ini
+   [project.scripts]
+   intelmq.bots.collectors.awesome.special = "awesome_bots.special:BOT.run"
+```
+
+Once you have installed your package, you can run ``intelmqctl list bots`` to check if your bot was
+properly registered.

intelmq/bin/intelmqctl.py

@@ -931,7 +931,7 @@ def check(self, no_connections=False, check_executables=True):
             if bot_id != 'global':
                 # importable module
                 try:
-                    bot_module = importlib.import_module(bot_config['module'])
+                    bot_module = importlib.import_module(utils.get_bot_module_name(bot_config['module']))
                 except ImportError as exc:
                     check_logger.error('Incomplete installation: Bot %r not importable: %r.', bot_id, exc)
                     retval = 1

intelmq/bots/experts/jinja/expert.py

@@ -27,31 +27,33 @@ class JinjaExpertBot(ExpertBot):
           extra.somejinjaoutput: file:///etc/intelmq/somejinjatemplate.j2
     """
 
-    fields: Dict[str, Union[str, Template]] = {}
+    fields: Dict[str, str] = {}
+    _templates: Dict[str, Union[str, Template]] = {}
     overwrite: bool = False
 
     def init(self):
         if not Template:
             raise MissingDependencyError("jinja2")
 
         for field, template in self.fields.items():
-            if template.startswith("file:///"):
-                templatefile = pathlib.Path(template[7:])
-                if templatefile.exists() and os.access(templatefile, os.R_OK):
-                    self.fields[field] = templatefile.read_text()
-                else:
-                    raise ValueError(f"Jinja Template {templatefile} does not exist or is not readable.")
+            if not template.startswith("file:///"):
+                continue
+
+            templatefile = pathlib.Path(template[7:])
+            if not (templatefile.exists() and os.access(templatefile, os.R_OK)):
+                raise ValueError(f"Jinja Template {templatefile} does not exist or is not readable.")
+            self.fields[field] = templatefile.read_text()
 
         for field, template in self.fields.items():
             try:
-                self.fields[field] = Template(template)
+                self._templates[field] = Template(template)
             except TemplateError as msg:
                 raise ValueError(f"Error parsing Jinja Template for '{field}': {msg}")
 
     def process(self):
         msg = self.receive_message()
 
-        for field, template in self.fields.items():
+        for field, template in self._templates.items():
             msg.add(field, template.render(msg=msg), overwrite=self.overwrite)
 
         self.send_message(msg)

intelmq/lib/bot_debugger.py

@@ -50,7 +50,7 @@ def __init__(self, runtime_configuration, bot_id, run_subcommand=None, console_t
         self.dryrun = dryrun
         self.msg = msg
         self.show = show
-        module = import_module(self.runtime_configuration['module'])
+        module = import_module(utils.get_bot_module_name(self.runtime_configuration['module']))
 
         if loglevel:
             self.leverageLogger(loglevel)

intelmq/lib/utils.py

@@ -34,7 +34,7 @@
 import textwrap
 import traceback
 import zipfile
-from pathlib import Path
+from sys import version_info
 from typing import (Any, Callable, Dict, Generator, Iterator, Optional,
                     Sequence, Union)
 
@@ -43,7 +43,6 @@
 import dns.version
 import requests
 from dateutil.relativedelta import relativedelta
-from pkg_resources import resource_filename
 from ruamel.yaml import YAML
 from ruamel.yaml.scanner import ScannerError
 from termstyle import red
@@ -52,6 +51,12 @@
 from intelmq import RUNTIME_CONF_FILE
 from intelmq.lib.exceptions import DecodingError
 
+try:
+    from importlib.metadata import entry_points
+except ImportError:
+    from importlib_metadata import entry_points
+
+
 __all__ = ['base64_decode', 'base64_encode', 'decode', 'encode',
            'load_configuration', 'load_parameters', 'log', 'parse_logline',
            'reverse_readline', 'error_message_from_exc', 'parse_relative',
@@ -839,6 +844,27 @@ def file_name_from_response(response: requests.Response) -> str:
     return file_name
 
 
+def _get_console_entry_points():
+    # Select interface was introduced in Python 3.10 and newer importlib_metadata
+    entries = entry_points()
+    if hasattr(entries, "select"):
+        return entries.select(group="console_scripts")
+    return entries.get("console_scripts", [])  # it's a dict
+
+
+def get_bot_module_name(bot_name: str) -> str:
+    entries = entry_points()
+    if hasattr(entries, "select"):
+        entries = tuple(entries.select(name=bot_name, group="console_scripts"))
+    else:
+        entries = [entry for entry in entries.get("console_scripts", []) if entry.name == bot_name]
+
+    if not entries:
+        return None
+    else:
+        return entries[0].value.replace(":BOT.run", '')
+
+
 def list_all_bots() -> dict:
     """
     Compile a dictionary with all bots and their parameters.
@@ -860,13 +886,11 @@ def list_all_bots() -> dict:
     from intelmq.lib.bot import Bot  # noqa: prevents circular import
     bot_parameters = dir(Bot)
 
-    base_path = resource_filename('intelmq', 'bots')
-
-    botfiles = [botfile for botfile in pathlib.Path(base_path).glob('**/*.py') if botfile.is_file() and botfile.name != '__init__.py']
-    for file in botfiles:
-        file = Path(file.as_posix().replace(base_path, 'intelmq/bots'))
+    bot_entrypoints = filter(lambda entry: entry.name.startswith("intelmq.bots."), _get_console_entry_points())
+    for bot in bot_entrypoints:
         try:
-            mod = importlib.import_module('.'.join(file.with_suffix('').parts))
+            module_name = bot.value.replace(":BOT.run", '')
+            mod = importlib.import_module(module_name)
         except SyntaxError:
             # Skip invalid bots
             continue
@@ -884,7 +908,7 @@ def list_all_bots() -> dict:
             for bot_type in ['CollectorBot', 'ParserBot', 'ExpertBot', 'OutputBot', 'Bot']:
                 name = name.replace(bot_type, '')
 
-            bots[file.parts[2].capitalize()[:-1]][name] = {
+            bots[module_name.split('.')[2].capitalize()[:-1]][name] = {
                 "module": mod.__name__,
                 "description": "Missing description" if not getattr(mod.BOT, '__doc__', None) else textwrap.dedent(mod.BOT.__doc__).strip(),
                 "parameters": keys,

intelmq/tests/bin/test_intelmqctl.py

@@ -125,6 +125,18 @@ def test_check_handles_syntaxerror_when_importing_bots(self):
         self.assertIsNotNone(
             next(filter(lambda l: "SyntaxError in bot 'test-bot'" in l, captured.output)))
 
+    @skip_installation()
+    @mock.patch.object(utils, "get_bot_module_name", mock.Mock(return_value="mocked-module"))
+    def test_check_imports_real_bot_module(self):
+        self._load_default_harmonization()
+        self._extend_config(self.tmp_runtime, self.BOT_CONFIG)
+
+        # raise SyntaxError to stop checking after import
+        with mock.patch.object(ctl.importlib, "import_module", mock.Mock(side_effect=SyntaxError)) as import_mock:
+            self.intelmqctl.check(no_connections=True, check_executables=False)
+
+        import_mock.assert_called_once_with("mocked-module")
+
 
 if __name__ == '__main__':  # pragma: nocover
     unittest.main()