diff --git a/CHANGELOG.md b/CHANGELOG.md index 14362fd6f..85c138e9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -186,6 +186,7 @@ Update allowed classification fields to 2020-01-28 version (#1409, #1476). - Handle empty "fieldnames" parameter by sending no attachment (PR#1932 by Sebastian Wagner). ### Documentation +- `dev/data-harmonization` renamed to `dev/data-format` (by Sebastian Waldbauer) - Feeds: - Fixed Abuse.ch Feodotracker Browse parser configuration (PR#1941 by Sebastian Wagner fixes #1938). diff --git a/intelmq/__init__.py b/intelmq/__init__.py index eebbddb67..0fb77e825 100644 --- a/intelmq/__init__.py +++ b/intelmq/__init__.py @@ -31,8 +31,9 @@ DEFAULT_LOGGING_LEVEL = "INFO" HARMONIZATION_CONF_FILE = os.path.join(CONFIG_DIR, "harmonization.conf") -PIPELINE_CONF_FILE = os.path.join(CONFIG_DIR, "pipeline.conf") -RUNTIME_CONF_FILE = os.path.join(CONFIG_DIR, "runtime.conf") -BOTS_FILE = os.path.join(CONFIG_DIR, "BOTS") +RUNTIME_CONF_FILE = os.path.join(CONFIG_DIR, "runtime.yaml") +old_runtime_conf_file = pathlib.Path(RUNTIME_CONF_FILE).with_suffix('.conf') +if not pathlib.Path(RUNTIME_CONF_FILE).exists() and old_runtime_conf_file.exists(): + old_runtime_conf_file.rename(RUNTIME_CONF_FILE) STATE_FILE_PATH = os.path.abspath(os.path.join(VAR_STATE_PATH, '../state.json')) diff --git a/intelmq/bots/outputs/smtp/output.py b/intelmq/bots/outputs/smtp/output.py index eb1b97acf..5a7e285f6 100644 --- a/intelmq/bots/outputs/smtp/output.py +++ b/intelmq/bots/outputs/smtp/output.py @@ -35,8 +35,11 @@ def init(self): self.smtp_class = smtplib.SMTP_SSL else: self.smtp_class = smtplib.SMTP - self.starttls = getattr(self.parameters, 'starttls', False) - self.fieldnames = getattr(self.parameters, 'fieldnames') + if self.http_verify_cert and self.smtp_class is smtplib.SMTP_SSL: + self.kwargs = {'context': ssl.create_default_context()} + else: + self.kwargs = {} + if self.fieldnames and isinstance(self.fieldnames, str): self.fieldnames = self.fieldnames.split(',') @@ -64,13 +67,13 @@ def process(self): self.logger.debug('Authenticating against SMTP server.') smtp.login(user=self.smtp_username, password=self.smtp_password) msg = MIMEMultipart() - if self.parameters.text: - msg.attach(MIMEText(self.parameters.text.format(ev=event))) + if self.text is not None: + msg.attach(MIMEText(self.text.format(ev=event))) if self.fieldnames: msg.attach(MIMEText(attachment, 'csv')) - msg['Subject'] = self.parameters.subject.format(ev=event) - msg['From'] = self.parameters.mail_from.format(ev=event) - msg['To'] = self.parameters.mail_to.format(ev=event) + msg['Subject'] = self.subject.format(ev=event) + msg['From'] = self.mail_from.format(ev=event) + msg['To'] = self.mail_to.format(ev=event) recipients = [recipient for recipient in self.mail_to.format(ev=event).split(',')] diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index b10886f9b..adb3ced13 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -2647,7 +2647,7 @@ def convert_date_utc(value: str) -> Optional[str]: ('extra.', 'event_id', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'malicious code', + 'classification.taxonomy': 'malicious-code', 'classification.type': 'infected-system', }, } @@ -2694,7 +2694,7 @@ def convert_date_utc(value: str) -> Optional[str]: ('extra.', 'http_referer', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'malicious code', + 'classification.taxonomy': 'malicious-code', 'classification.type': 'infected-system', 'protocol.application': 'http', }, diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index f19e384f0..a0878a12a 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -33,6 +33,9 @@ 'v230_deprecations', 'v230_feed_changes', 'v233_feodotracker_browse', + 'v300_bots_file_removal', + 'v300_defaults_file_removal', + 'v300_pipeline_file_removal' ] @@ -574,6 +577,26 @@ def v230_feed_changes(defaults, runtime, harmonization, dry_run): return messages + ' Remove affected bots yourself.' if messages else changed, defaults, runtime, harmonization +def v233_feodotracker_browse(defaults, runtime, harmonization, dry_run): + """ + Migrate Abuse.ch Feodotracker Browser feed parsing parameters + """ + changed = None + old_feodo_columns = 'time.source,source.ip,malware.name,status,extra.SBL,source.as_name,source.geolocation.cc' + old_ignore_values = ',,,,Not listed,,' + for bot_id, bot in runtime.items(): + # The parameters can be given as string or list of strings + if (bot["module"] == "intelmq.bots.parsers.html_table.parser" and 'feodo' in bot_id.lower() and + "columns" in bot["parameters"] and "ignore_values" in bot["parameters"] and + (bot["parameters"]["columns"] == old_feodo_columns or bot["parameters"]["columns"] == old_feodo_columns.split(',')) and + (bot["parameters"]["ignore_values"] == old_ignore_values or bot["parameters"]["ignore_values"] == old_ignore_values.split(','))): + bot["parameters"]["columns"] = 'time.source,source.ip,malware.name,status,source.as_name,source.geolocation.cc' + bot["parameters"]['ignore_values'] = ',,,,,' + changed = True + messages = ' '.join(messages) + return messages if messages else changed, defaults, runtime, harmonization + + def v233_feodotracker_browse(defaults, runtime, harmonization, dry_run): """ Migrate Abuse.ch Feodotracker Browser feed parsing parameters @@ -593,6 +616,37 @@ def v233_feodotracker_browse(defaults, runtime, harmonization, dry_run): return changed, defaults, runtime, harmonization +def v300_pipeline_file_removal(defaults, runtime, harmonization, dry_run): + """ + Remove the pipeline.conf file + """ + changed = None + messages = [] + pipeline_file = Path(CONFIG_DIR) / "pipeline.conf" + if pipeline_file.exists(): + pipelines = load_configuration(pipeline_file) + for bot in runtime: + if bot in pipelines: + if 'destination-queues' in pipelines[bot]: + destination_queues = pipelines[bot]['destination-queues'] + if isinstance(destination_queues, dict): + runtime[bot]['parameters']['destination_queues'] = destination_queues + if isinstance(destination_queues, list): + runtime[bot]['parameters']['destination_queues'] = {'_default': destination_queues} + if isinstance(destination_queues, str): + runtime[bot]['parameters']['destination_queues'] = {'_default': [destination_queues]} + if 'source-queue' in pipelines[bot]: + if pipelines[bot]['source-queue'] != f"{bot}-queue": + runtime[bot]['parameters']['source_queue'] = pipelines[bot]['source-queue'] + if dry_run: + print(f'Would now remove file {pipeline_file!r}.') + else: + pipeline_file.unlink() + changed = True + messages = ' '.join(messages) + return messages if messages else changed, defaults, runtime, harmonization + + UPGRADES = OrderedDict([ ((1, 0, 0, 'dev7'), (v100_dev7_modify_syntax, )), ((1, 1, 0), (v110_shadowserver_feednames, v110_deprecations)), @@ -614,6 +668,7 @@ def v233_feodotracker_browse(defaults, runtime, harmonization, dry_run): ((2, 3, 1), ()), ((2, 3, 2), ()), ((2, 3, 3), (v233_feodotracker_browse, )), + ((3, 0, 0), (v300_bots_file_removal, v300_defaults_file_removal, v300_pipeline_file_removal, )) ]) ALWAYS = (harmonization, ) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py index ad446adeb..c3abb09bd 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py @@ -23,7 +23,7 @@ "extra.file_name": "2019-01-01-event4_sinkhole.csv", } EVENTS = [{'__type': 'Event', - 'classification.taxonomy': 'malicious code', + 'classification.taxonomy': 'malicious-code', 'classification.type': 'infected-system', 'destination.asn': 28753, 'destination.geolocation.cc': 'DE', @@ -49,7 +49,7 @@ }, {'__type': 'Event', 'classification.identifier': 'b68-zeroaccess-2-32bit', - 'classification.taxonomy': 'malicious code', + 'classification.taxonomy': 'malicious-code', 'classification.type': 'infected-system', 'destination.asn': 8075, 'destination.geolocation.cc': 'SG', @@ -76,7 +76,7 @@ }, {'__type': 'Event', 'classification.identifier': 'b68-zeroaccess-2-32bit', - 'classification.taxonomy': 'malicious code', + 'classification.taxonomy': 'malicious-code', 'classification.type': 'infected-system', 'destination.asn': 8075, 'destination.geolocation.cc': 'HK', diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py index 9e8006abf..d694a49a6 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py @@ -25,7 +25,7 @@ EVENTS = [{'__type': 'Event', 'feed.name': 'HTTP Sinkhole IPv4', 'classification.identifier': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious code', + 'classification.taxonomy': 'malicious-code', 'classification.type': 'infected-system', 'destination.asn': 6939, 'destination.fqdn': 'differentia.ru', @@ -52,7 +52,7 @@ {'__type': 'Event', 'feed.name': 'HTTP Sinkhole IPv4', 'classification.identifier': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious code', + 'classification.taxonomy': 'malicious-code', 'classification.type': 'infected-system', 'destination.asn': 6939, 'destination.fqdn': 'differentia.ru', @@ -81,7 +81,7 @@ {'__type': 'Event', 'feed.name': 'HTTP Sinkhole IPv4', 'classification.identifier': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious code', + 'classification.taxonomy': 'malicious-code', 'classification.type': 'infected-system', 'destination.asn': 6939, 'destination.fqdn': 'disorderstatus.ru', @@ -109,7 +109,7 @@ {'__type': 'Event', 'feed.name': 'HTTP Sinkhole IPv4', 'classification.identifier': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious code', + 'classification.taxonomy': 'malicious-code', 'classification.type': 'infected-system', 'destination.asn': 6939, 'destination.fqdn': 'differentia.ru', @@ -137,7 +137,7 @@ {'__type': 'Event', 'feed.name': 'HTTP Sinkhole IPv4', 'classification.identifier': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious code', + 'classification.taxonomy': 'malicious-code', 'classification.type': 'infected-system', 'destination.asn': 6939, 'destination.fqdn': 'differentia.ru', diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py index 62420cdc1..0e3e3cf8a 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: 2021 Mikk Margus Möll +# +# SPDX-License-Identifier: AGPL-3.0-or-later # -*- coding: utf-8 -*- import os