Merge branch 'develop' into shadowserver-dynamic-config

certtools · Dec 12, 2023 · 0c0cb68 · 0c0cb68
2 parents 7a7a6a6 + 4075768
commit 0c0cb68
Show file tree

Hide file tree

Showing 5 changed files with 68 additions and 8 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -32,6 +32,7 @@
   which provides certain common STOMP-bot-specific operations, factored out from
   `intelmq.bots.collectors.stomp.collector` and `intelmq.bots.outputs.stomp.output`
   (PR#2408 and PR#2414 by Jan Kaliszewski).
+- `intelmq.lib.upgrades`: Replace deprecated instances of `url2fqdn` experts by the new `url` expert in runtime configuration (PR#2432 by Sebastian Wagner).
 
 ### Development
 - Makefile: Add codespell and test commands (PR#2425 by Sebastian Wagner).
@@ -84,6 +85,8 @@
     of necessary file(s)).
   - Add `stomp.py` version check (raise `MissingDependencyError` if not `>=4.1.12`).
   - Minor fixes/improvements and some refactoring (see also above: *Core*...).
+- `intelmq.bots.outputs.stomp.output` (PR#2423 by Kamil Mankowski):
+  - Try to reconnect on `NotConnectedException`.
 
 ### Documentation
 - Add a readthedocs configuration file to fix the build fail (PR#2403 by Sebastian Wagner).

diff --git a/intelmq/bots/outputs/stomp/output.py b/intelmq/bots/outputs/stomp/output.py
@@ -76,8 +76,12 @@ def process(self):
 
         body = self.export_event(event)
 
-        self._conn.send(body=body,
-                        destination=self.exchange)
+        try:
+            self._conn.send(body=body, destination=self.exchange)
+        except stomp.exception.NotConnectedException:
+            self.logger.warning("Detected connection error, trying to reestablish it.")
+            self.connect()
+            raise  # Fallback to default retry
         self.acknowledge_message()
 
     @classmethod

diff --git a/intelmq/etc/runtime.yaml b/intelmq/etc/runtime.yaml
@@ -156,16 +156,16 @@ taxonomy-expert:
   parameters:
     destination_queues:
       _default:
-      - url2fqdn-expert-queue
+      - url-expert-queue
   run_mode: continuous
-url2fqdn-expert:
-  bot_id: url2fqdn-expert
-  description: url2fqdn is the bot responsible to parsing the fqdn from the url.
+url-expert:
+  bot_id: url-expert
+  description: Extract additional information for the URL
   enabled: true
   group: Expert
   groupname: experts
-  module: intelmq.bots.experts.url2fqdn.expert
-  name: URL2FQDN
+  module: intelmq.bots.experts.url.expert
+  name: url
   parameters:
     destination_queues:
       _default:

diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py
@@ -39,6 +39,7 @@
            'v310_feed_changes',
            'v310_shadowserver_feednames',
            'v320_update_turris_greylist_url',
+           'v322_url_replacement',
            ]
 
 
@@ -879,6 +880,24 @@ def v320_update_turris_greylist_url(configuration, harmonization, dry_run, **kwa
     return ' '.join(messages) if messages else None, configuration, harmonization
 
 
+def v322_url_replacement(configuration, harmonization, dry_run, **kwargs):
+    """
+    Replace deprecated url2fqdn expert with url expert.
+    """
+    changed = None
+    for bot_id, bot in configuration.items():
+        if bot_id == 'global':
+            continue
+        if bot["module"] == "intelmq.bots.experts.url2fqdn.expert":
+            configuration[bot_id]["module"] = "intelmq.bots.experts.url.expert"
+            if "parameters" not in configuration[bot_id]:
+                configuration[bot_id]["parameters"] = {}
+            # skip all fields except the fqdn field for backwards compatibility
+            configuration[bot_id]["parameters"]["skip_fields"] = ["source.ip", "source.port", "source.urlpath", "source.account", "destination.ip", "destination.port", "destination.urlpath", "destination.account", "protocol.application", "protocol.transport"]
+            changed = True
+    return changed, configuration, harmonization
+
+
 UPGRADES = OrderedDict([
     ((1, 0, 0, 'dev7'), (v100_dev7_modify_syntax,)),
     ((1, 1, 0), (v110_shadowserver_feednames, v110_deprecations)),
@@ -905,6 +924,7 @@ def v320_update_turris_greylist_url(configuration, harmonization, dry_run, **kwa
     ((3, 0, 2), ()),
     ((3, 1, 0), (v310_feed_changes, v310_shadowserver_feednames)),
     ((3, 2, 0), (v320_update_turris_greylist_url,)),
+    ((3, 2, 2), (v322_url_replacement, )),
 ])
 
 ALWAYS = (harmonization,)
diff --git a/intelmq/tests/lib/test_upgrades.py b/intelmq/tests/lib/test_upgrades.py
@@ -550,6 +550,29 @@
         "parameters": {}
     }
 }
+V322_URL2FQN_IN = {
+    "global": {},
+    "url2fqdn-expert": {
+        "module": "intelmq.bots.experts.url2fqdn.expert",
+        "parameters": {
+        }
+    },
+}
+V322_URL2FQN_IN_1 = {
+    "global": {},
+    "url2fqdn-expert": {
+        "module": "intelmq.bots.experts.url2fqdn.expert",
+    },
+}
+V322_URL2FQN_OUT = {
+    "global": {},
+    "url2fqdn-expert": {
+        "module": "intelmq.bots.experts.url.expert",
+        "parameters": {
+            "skip_fields": ["source.ip", "source.port", "source.urlpath", "source.account", "destination.ip", "destination.port", "destination.urlpath", "destination.account", "protocol.application", "protocol.transport"]
+        },
+    },
+}
 
 
 def generate_function(function):
@@ -762,6 +785,16 @@ def test_v310_feed_changes(self):
                          result[0])
         self.assertEqual(V310_FEED_CHANGES, result[1])
 
+    def test_v322_url_replacement(self):
+        """ Test v322_url_replacement """
+        result = upgrades.v322_url_replacement(V322_URL2FQN_IN, {}, False)
+        self.assertTrue(result[0])
+        self.assertEqual(V322_URL2FQN_OUT, result[1])
+
+        result = upgrades.v322_url_replacement(V322_URL2FQN_IN_1, {}, False)
+        self.assertTrue(result[0])
+        self.assertEqual(V322_URL2FQN_OUT, result[1])
+
 
 for name in upgrades.__all__:
     setattr(TestUpgradeLib, 'test_function_%s' % name,