Merge branch 'waldbauer/msgpack-poc' of https://github.com/certtools/…

…intelmq into waldbauer/msgpack-poc
certtools · May 31, 2022 · 5697faf · 5697faf
2 parents ef8369f + 40e4ae1
commit 5697faf
Show file tree

Hide file tree

Showing 8 changed files with 83 additions and 20 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -186,6 +186,7 @@ Update allowed classification fields to 2020-01-28 version (#1409, #1476).
   - Handle empty "fieldnames" parameter by sending no attachment (PR#1932 by Sebastian Wagner).
 
 ### Documentation
+- `dev/data-harmonization` renamed to `dev/data-format` (by Sebastian Waldbauer)
 - Feeds:
   - Fixed Abuse.ch Feodotracker Browse parser configuration (PR#1941 by Sebastian Wagner fixes #1938).
 

diff --git a/intelmq/__init__.py b/intelmq/__init__.py
@@ -31,8 +31,9 @@
 
 DEFAULT_LOGGING_LEVEL = "INFO"
 HARMONIZATION_CONF_FILE = os.path.join(CONFIG_DIR, "harmonization.conf")
-PIPELINE_CONF_FILE = os.path.join(CONFIG_DIR, "pipeline.conf")
-RUNTIME_CONF_FILE = os.path.join(CONFIG_DIR, "runtime.conf")
-BOTS_FILE = os.path.join(CONFIG_DIR, "BOTS")
+RUNTIME_CONF_FILE = os.path.join(CONFIG_DIR, "runtime.yaml")
+old_runtime_conf_file = pathlib.Path(RUNTIME_CONF_FILE).with_suffix('.conf')
+if not pathlib.Path(RUNTIME_CONF_FILE).exists() and old_runtime_conf_file.exists():
+    old_runtime_conf_file.rename(RUNTIME_CONF_FILE)
 STATE_FILE_PATH = os.path.abspath(os.path.join(VAR_STATE_PATH,
                                                '../state.json'))
diff --git a/intelmq/bots/outputs/smtp/output.py b/intelmq/bots/outputs/smtp/output.py
@@ -35,8 +35,11 @@ def init(self):
             self.smtp_class = smtplib.SMTP_SSL
         else:
             self.smtp_class = smtplib.SMTP
-        self.starttls = getattr(self.parameters, 'starttls', False)
-        self.fieldnames = getattr(self.parameters, 'fieldnames')
+        if self.http_verify_cert and self.smtp_class is smtplib.SMTP_SSL:
+            self.kwargs = {'context': ssl.create_default_context()}
+        else:
+            self.kwargs = {}
+
         if self.fieldnames and isinstance(self.fieldnames, str):
             self.fieldnames = self.fieldnames.split(',')
 
@@ -64,13 +67,13 @@ def process(self):
                 self.logger.debug('Authenticating against SMTP server.')
                 smtp.login(user=self.smtp_username, password=self.smtp_password)
             msg = MIMEMultipart()
-            if self.parameters.text:
-                msg.attach(MIMEText(self.parameters.text.format(ev=event)))
+            if self.text is not None:
+                msg.attach(MIMEText(self.text.format(ev=event)))
             if self.fieldnames:
                 msg.attach(MIMEText(attachment, 'csv'))
-            msg['Subject'] = self.parameters.subject.format(ev=event)
-            msg['From'] = self.parameters.mail_from.format(ev=event)
-            msg['To'] = self.parameters.mail_to.format(ev=event)
+            msg['Subject'] = self.subject.format(ev=event)
+            msg['From'] = self.mail_from.format(ev=event)
+            msg['To'] = self.mail_to.format(ev=event)
             recipients = [recipient
                           for recipient
                           in self.mail_to.format(ev=event).split(',')]

diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py
@@ -2647,7 +2647,7 @@ def convert_date_utc(value: str) -> Optional[str]:
         ('extra.', 'event_id', validate_to_none),
     ],
     'constant_fields': {
-        'classification.taxonomy': 'malicious code',
+        'classification.taxonomy': 'malicious-code',
         'classification.type': 'infected-system',
     },
 }
@@ -2694,7 +2694,7 @@ def convert_date_utc(value: str) -> Optional[str]:
         ('extra.', 'http_referer', validate_to_none),
     ],
     'constant_fields': {
-        'classification.taxonomy': 'malicious code',
+        'classification.taxonomy': 'malicious-code',
         'classification.type': 'infected-system',
         'protocol.application': 'http',
     },

diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py
@@ -33,6 +33,9 @@
            'v230_deprecations',
            'v230_feed_changes',
            'v233_feodotracker_browse',
+           'v300_bots_file_removal',
+           'v300_defaults_file_removal',
+           'v300_pipeline_file_removal'
            ]
 
 
@@ -574,6 +577,26 @@ def v230_feed_changes(defaults, runtime, harmonization, dry_run):
     return messages + ' Remove affected bots yourself.' if messages else changed, defaults, runtime, harmonization
 
 
+def v233_feodotracker_browse(defaults, runtime, harmonization, dry_run):
+    """
+    Migrate Abuse.ch Feodotracker Browser feed parsing parameters
+    """
+    changed = None
+    old_feodo_columns = 'time.source,source.ip,malware.name,status,extra.SBL,source.as_name,source.geolocation.cc'
+    old_ignore_values = ',,,,Not listed,,'
+    for bot_id, bot in runtime.items():
+        # The parameters can be given as string or list of strings
+        if (bot["module"] == "intelmq.bots.parsers.html_table.parser" and 'feodo' in bot_id.lower() and
+                "columns" in bot["parameters"] and "ignore_values" in bot["parameters"] and
+                (bot["parameters"]["columns"] == old_feodo_columns or bot["parameters"]["columns"] == old_feodo_columns.split(',')) and
+                (bot["parameters"]["ignore_values"] == old_ignore_values or bot["parameters"]["ignore_values"] == old_ignore_values.split(','))):
+            bot["parameters"]["columns"] = 'time.source,source.ip,malware.name,status,source.as_name,source.geolocation.cc'
+            bot["parameters"]['ignore_values'] = ',,,,,'
+            changed = True
+    messages = ' '.join(messages)
+    return messages if messages else changed, defaults, runtime, harmonization
+
+
 def v233_feodotracker_browse(defaults, runtime, harmonization, dry_run):
     """
     Migrate Abuse.ch Feodotracker Browser feed parsing parameters
@@ -593,6 +616,37 @@ def v233_feodotracker_browse(defaults, runtime, harmonization, dry_run):
     return changed, defaults, runtime, harmonization
 
 
+def v300_pipeline_file_removal(defaults, runtime, harmonization, dry_run):
+    """
+    Remove the pipeline.conf file
+    """
+    changed = None
+    messages = []
+    pipeline_file = Path(CONFIG_DIR) / "pipeline.conf"
+    if pipeline_file.exists():
+        pipelines = load_configuration(pipeline_file)
+        for bot in runtime:
+            if bot in pipelines:
+                if 'destination-queues' in pipelines[bot]:
+                    destination_queues = pipelines[bot]['destination-queues']
+                    if isinstance(destination_queues, dict):
+                        runtime[bot]['parameters']['destination_queues'] = destination_queues
+                    if isinstance(destination_queues, list):
+                        runtime[bot]['parameters']['destination_queues'] = {'_default': destination_queues}
+                    if isinstance(destination_queues, str):
+                        runtime[bot]['parameters']['destination_queues'] = {'_default': [destination_queues]}
+                if 'source-queue' in pipelines[bot]:
+                    if pipelines[bot]['source-queue'] != f"{bot}-queue":
+                        runtime[bot]['parameters']['source_queue'] = pipelines[bot]['source-queue']
+        if dry_run:
+            print(f'Would now remove file {pipeline_file!r}.')
+        else:
+            pipeline_file.unlink()
+        changed = True
+    messages = ' '.join(messages)
+    return messages if messages else changed, defaults, runtime, harmonization
+
+
 UPGRADES = OrderedDict([
     ((1, 0, 0, 'dev7'), (v100_dev7_modify_syntax, )),
     ((1, 1, 0), (v110_shadowserver_feednames, v110_deprecations)),
@@ -614,6 +668,7 @@ def v233_feodotracker_browse(defaults, runtime, harmonization, dry_run):
     ((2, 3, 1), ()),
     ((2, 3, 2), ()),
     ((2, 3, 3), (v233_feodotracker_browse, )),
+    ((3, 0, 0), (v300_bots_file_removal, v300_defaults_file_removal, v300_pipeline_file_removal, ))
 ])
 
 ALWAYS = (harmonization, )
diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py
@@ -23,7 +23,7 @@
                   "extra.file_name": "2019-01-01-event4_sinkhole.csv",
                   }
 EVENTS = [{'__type': 'Event',
-           'classification.taxonomy': 'malicious code',
+           'classification.taxonomy': 'malicious-code',
            'classification.type': 'infected-system',
            'destination.asn': 28753,
            'destination.geolocation.cc': 'DE',
@@ -49,7 +49,7 @@
            },
           {'__type': 'Event',
            'classification.identifier': 'b68-zeroaccess-2-32bit',
-           'classification.taxonomy': 'malicious code',
+           'classification.taxonomy': 'malicious-code',
            'classification.type': 'infected-system',
            'destination.asn': 8075,
            'destination.geolocation.cc': 'SG',
@@ -76,7 +76,7 @@
            },
           {'__type': 'Event',
            'classification.identifier': 'b68-zeroaccess-2-32bit',
-           'classification.taxonomy': 'malicious code',
+           'classification.taxonomy': 'malicious-code',
            'classification.type': 'infected-system',
            'destination.asn': 8075,
            'destination.geolocation.cc': 'HK',

diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py
@@ -25,7 +25,7 @@
 EVENTS = [{'__type': 'Event',
            'feed.name': 'HTTP Sinkhole IPv4',
            'classification.identifier': 'avalanche-andromeda',
-           'classification.taxonomy': 'malicious code',
+           'classification.taxonomy': 'malicious-code',
            'classification.type': 'infected-system',
            'destination.asn': 6939,
            'destination.fqdn': 'differentia.ru',
@@ -52,7 +52,7 @@
           {'__type': 'Event',
            'feed.name': 'HTTP Sinkhole IPv4',
            'classification.identifier': 'avalanche-andromeda',
-           'classification.taxonomy': 'malicious code',
+           'classification.taxonomy': 'malicious-code',
            'classification.type': 'infected-system',
            'destination.asn': 6939,
            'destination.fqdn': 'differentia.ru',
@@ -81,7 +81,7 @@
           {'__type': 'Event',
            'feed.name': 'HTTP Sinkhole IPv4',
            'classification.identifier': 'avalanche-andromeda',
-           'classification.taxonomy': 'malicious code',
+           'classification.taxonomy': 'malicious-code',
            'classification.type': 'infected-system',
            'destination.asn': 6939,
            'destination.fqdn': 'disorderstatus.ru',
@@ -109,7 +109,7 @@
           {'__type': 'Event',
            'feed.name': 'HTTP Sinkhole IPv4',
            'classification.identifier': 'avalanche-andromeda',
-           'classification.taxonomy': 'malicious code',
+           'classification.taxonomy': 'malicious-code',
            'classification.type': 'infected-system',
            'destination.asn': 6939,
            'destination.fqdn': 'differentia.ru',
@@ -137,7 +137,7 @@
           {'__type': 'Event',
            'feed.name': 'HTTP Sinkhole IPv4',
            'classification.identifier': 'avalanche-andromeda',
-           'classification.taxonomy': 'malicious code',
+           'classification.taxonomy': 'malicious-code',
            'classification.type': 'infected-system',
            'destination.asn': 6939,
            'destination.fqdn': 'differentia.ru',

diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py
@@ -1,3 +1,6 @@
+# SPDX-FileCopyrightText: 2021 Mikk Margus Möll <[email protected]>
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
 # -*- coding: utf-8 -*-
 
 import os