Merge branch 'develop' into makefile-codestyle

.github/workflows/python/github.py

@@ -31,20 +31,20 @@ def __init__(self):
 
     def get_reviews(self):
         """ Get a list of reviews on a Github pull request as json object """
-        reviews = self.session.get(self.api + 'repos/{}/pulls/{}/reviews'.format(self.github_repository, self.pr_id))
+        reviews = self.session.get(self.api + f'repos/{self.github_repository}/pulls/{self.pr_id}/reviews')
         reviews.raise_for_status()
         return reviews.json()
 
     def update_review(self, review_id, body):
         """ Update a review given by `review_id` and set its body to `body` """
         payload = {'body': body}
-        resp = self.session.put(self.api + 'repos/{}/pulls/{}/reviews/{}'.format(self.github_repository, self.pr_id, review_id), json=payload)
+        resp = self.session.put(self.api + f'repos/{self.github_repository}/pulls/{self.pr_id}/reviews/{review_id}', json=payload)
         resp.raise_for_status()
         return resp.json()
 
     def post_review(self, body):
         """ Post a pull request review containing `body` and requesting changes """
         payload = {'body': body, 'event': "REQUEST_CHANGES"}
-        resp = self.session.post(self.api + 'repos/{}/pulls/{}/reviews'.format(self.github_repository, self.pr_id), json=payload)
+        resp = self.session.post(self.api + f'repos/{self.github_repository}/pulls/{self.pr_id}/reviews', json=payload)
         resp.raise_for_status()
         return resp.json()

.github/workflows/python/pycodestyle_comment.py

@@ -34,7 +34,7 @@ def style_error_format(style_error_list) -> str:
     """ Format the list of pycodestyle errors and return them a one string. """
     ret = ''
     for error in style_error_list:
-        ret += '* {}\n'.format(error)
+        ret += f'* {error}\n'
     return ret
 
 
@@ -45,7 +45,7 @@ def style_error_format(style_error_list) -> str:
         style_errors = list_style_errors()
 
         if style_errors:
-            print("Found {} errors.".format(len(style_errors)))
+            print(f"Found {len(style_errors)} errors.")
 
         gh = github.Github()
 

.github/workflows/scripts/setup-full.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# SPDX-FileCopyrightText: 2020 Birger Schacht
+# SPDX-FileCopyrightText: 2020 Birger Schacht, 2024 Institute for Common Good Technology
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 set -x
@@ -14,6 +14,14 @@ echo -e '-XX:+DisableExplicitGC\n-Djdk.io.permissionsUseCanonicalPath=true\n-Dlo
 sudo chown -R elasticsearch:elasticsearch /etc/default/elasticsearch
 sudo systemctl start elasticsearch
 
+sudo apt update
+if [ $python_version == '3.8' ]; then
+	# for pymssql there are no wheels for 3.8 https://github.com/certtools/intelmq/issues/2539
+	DEBIAN_FRONTEND="noninteractive" sudo -E apt install -y build-essential freetds-dev libssl-dev libkrb5-dev
+fi
+# for psql (used below)
+DEBIAN_FRONTEND="noninteractive" sudo -E apt install -y postgresql-client-14
+
 # Install the dependencies of all the bots
 pip install wheel
 for file in intelmq/bots/*/*/REQUIREMENTS.txt; do
@@ -30,7 +38,16 @@ done
 # Setup sudo and install intelmq
 sudo sed -i '/^Defaults\tsecure_path.*$/ d' /etc/sudoers
 sudo pip install .
-sudo intelmqsetup --skip-ownership
+
+intelmq_user_exists=$(getent passwd intelmq ||:)
+if [[ "$UID" -eq '0' && -z "$intelmq_user_exists" ]]; then
+	# create an unprivileged user, if currently running as root. Otherwise dropping privileges won't work
+	groupadd -r intelmq
+	useradd -r -d /var/lib/intelmq/ -c "user running intelmq" -g intelmq -s /bin/bash intelmq
+	sudo intelmqsetup
+else
+	sudo intelmqsetup --skip-ownership
+fi
 
 # Initialize the postgres database
 intelmq_psql_initdb

.github/workflows/unittests.yml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
         type: ['full', 'basic']
 
     services:
@@ -59,6 +59,7 @@ jobs:
         PGPORT: 5432
         PGUSER: intelmq
         PGPASSWORD: intelmq
+        python_version: ${{ matrix.python-version }}
       run: bash .github/workflows/scripts/setup-full.sh
 
     - name: Install test dependencies

CHANGELOG.md

@@ -12,13 +12,16 @@
 ### Configuration
 
 ### Core
+- Python 3.8 or newer is required (PR#2541 by Sebastian Wagner).
 
 ### Development
 
 ### Data Format
 
 ### Bots
 #### Collectors
+- `intelmq.bots.collectors.shadowserver.collector_reports_api.py`:
+  - Fixed behaviour if parameter `types` value is empty string, behave the same way as not set, not like no type.
 
 #### Parsers
 - `intelmq.bots.parsers.shadowserver._config`:
@@ -29,12 +32,20 @@
 #### Experts
 
 #### Outputs
+- `intelmq.bots.outputs.cif3.output`:
+  - The requirement can only be installed on Python version < 3.12.
+  - Add a check on the Python version and exit if incompatible.
+  - Add a deprecation warning (PR#2544 by Sebastian Wagner)
 
 ### Documentation
 
 ### Packaging
 
 ### Tests
+- Install build dependencies for `pymssql` on Python 3.8 as there are no wheels available for this Python version (PR#2542 by Sebastian Wagner).
+- Install `psql` explicitly for workflow support on other platforms such as act (PR#2542 by Sebastian Wagner).
+- Create intelmq user & group if running privileged to allow dropping privileges (PR#2542 by Sebastian Wagner).
+- `intelmq.tests.lib.test_pipeline.TestAmqp.test_acknowledge`: Also skip on Python 3.11 besides on 3.8 when running on CI (PR#2542 by Sebastian Wagner).
 
 ### Tools
 

NEWS.md

@@ -14,6 +14,12 @@ Please refer to the change log for a full list of changes.
 ---------------------------------
 
 ### Requirements
+Python 3.8 or newer is required.
+
+## Bots
+#### CIF 3 API
+The CIF 3 API Output bot is not compatible with Python version greater or equal to 3.12 and will be removed in the future due to lack of maintenance.
+See https://lists.cert.at/pipermail/intelmq-users/2024-December/000474.html for more information.
 
 ### Tools
 
@@ -36,7 +42,7 @@ No changes are required by administrators.
 ----------------------------------
 
 ### Documentation
-The documentation is now available at [docs.intelmq.org](https://docs.intelmq.org/). Documentation has been updated and restructured into User, Administrator and Developer Guide. It provides modern look with various quality of life improvements. Big thanks to to @gethvi. 
+The documentation is now available at [docs.intelmq.org](https://docs.intelmq.org/). Documentation has been updated and restructured into User, Administrator and Developer Guide. It provides modern look with various quality of life improvements. Big thanks to to @gethvi.
 We now have a slick, modern mkdocs based documentation. Please do check it out!
 
 
@@ -55,7 +61,7 @@ Shadowserver adds new scans on a nearly weekly basis. IntelMQ's release cycle an
 We therefore (thanks to @eslif2) move the shadowserver reports collector and parser to a new, dynamic system. It can:
 
  - fetch the shadowserver schema from shadowserver (https://interchange.shadowserver.org/intelmq/v1/schema)
- - dynamically collect new reports (see also https://docs.intelmq.org/latest/user/bots/?h=shadow#shadowserver-reports-api) 
+ - dynamically collect new reports (see also https://docs.intelmq.org/latest/user/bots/?h=shadow#shadowserver-reports-api)
  - parse the new reports
 
 **Note well**: if your IntelMQ system runs in an airgapped environment or if it may only reach out to specific IPs/sites, you should read the notes here:
@@ -86,7 +92,7 @@ Quite a few changes (thanks to Kamil, @gethvi) on AMQP
 ### General changes and bug fixes
 
 Digital Trust Center fixed a bug where the config was loaded twice in intelmqctl which created quite some speedups. Thanks!
-This speeds up IntelMQ API calls. 
+This speeds up IntelMQ API calls.
 
 ### Data Format
 

contrib/example-extension-package/setup.py

@@ -28,7 +28,7 @@
     entry_point = '.'.join(file.with_suffix('').parts)
     file = Path(str(file).replace('intelmq/bots', 'mybots/bots'))
     module = '.'.join(file.with_suffix('').parts)
-    BOTS.append('{0} = {1}:BOT.run'.format(entry_point, module))
+    BOTS.append(f'{entry_point} = {module}:BOT.run')
 
 setup(
     name='intelmq-example-extension',

intelmq/bots/collectors/rsync/collector_rsync.py

@@ -13,7 +13,7 @@
 from intelmq.lib.bot import CollectorBot
 
 
-class Time(object):
+class Time:
     def __init__(self, delta=None):
         """ Delta is a datetime.timedelta JSON string, ex: '{days=-1}'. """
         self.time = datetime.now()

intelmq/bots/collectors/shadowserver/collector_reports_api.py

@@ -33,7 +33,7 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin):
         reports (list):
             A list of strings or a comma-separated list of the mailing lists you want to process.
         types (list):
-            A list of strings or a string of comma-separated values with the names of reporttypes you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined').
+            A list of strings or a string of comma-separated values with the names of report types you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined').
     """
 
     country = None
@@ -48,6 +48,7 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin):
     redis_cache_ttl: int = 864000  # 10 days
     redis_cache_password: Optional[str] = None
     _report_list = []
+    _type_list = []
 
     def init(self):
         if not self.api_key:
@@ -62,7 +63,11 @@ def init(self):
         elif isinstance(self.reports, list):
             self._report_list = self.reports
         if isinstance(self.types, str):
-            self.types = self.types.split(',')
+            # if types is an empty string (or only contains whitespace), behave as if the parameter is not set and select all types
+            types = self.types.strip()
+            self._type_list = types.split(',') if types else []
+        elif isinstance(self.types, list):
+            self._type_list = self.types
         if self.country and self.country not in self._report_list:
             self.logger.warn("Deprecated parameter 'country' found. Please use 'reports' instead. The backwards-compatibility will be removed in IntelMQ version 4.0.0.")
             self._report_list.append(self.country)
@@ -111,8 +116,8 @@ def _reports_list(self, date=None):
             self.logger.debug('There was an error downloading the reports: %s', reports['error'])
             return None
 
-        if self.types:
-            reports = [report for report in reports if any(report['type'] == rtype for rtype in self.types)]
+        if self._type_list:
+            reports = [report for report in reports if any(report['type'] == rtype for rtype in self._type_list)]
         return reports
 
     def _report_download(self, reportid: str):

intelmq/bots/experts/modify/expert.py

@@ -17,10 +17,7 @@ def is_re_pattern(value):
     """
     Checks if the given value is a re compiled pattern
     """
-    if sys.version_info > (3, 7):
-        return isinstance(value, re.Pattern)
-    else:
-        return hasattr(value, "pattern")
+    return isinstance(value, re.Pattern)
 
 
 class MatchGroupMapping:

intelmq/bots/outputs/cif3/REQUIREMENTS.txt

@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# SPDX-FileCopyrightText: 2022 REN-ISAC
+# SPDX-FileCopyrightText: 2022 REN-ISAC, 2024 Institute for Common Good Technology
 
-cifsdk>=3.0.0rc4,<4.0
+cifsdk>=3.0.0rc4,<4.0; python_version < '3.12'

intelmq/bots/outputs/cif3/output.py

@@ -32,6 +32,7 @@
     JsonLib = None
 
 from datetime import datetime
+from sys import version_info
 from typing import Optional, List
 
 from intelmq.lib.bot import OutputBot
@@ -91,6 +92,9 @@ class CIF3OutputBot(OutputBot):
     _is_multithreadable = False
 
     def init(self):
+        raise DeprecationWarning("The CIFv3 API is deprecated and will be remove in IntelMQ version 4.0. See https://lists.cert.at/pipermail/intelmq-users/2024-December/000474.html")
+        if version_info >= (3, 12):
+            raise ValueError("This bot is not compatible with Python >= 3.12. See https://lists.cert.at/pipermail/intelmq-users/2024-December/000474.html")
         try:
             cifsdk_version = int(get_cifsdk_version().get('version').split('.')[0])
         except NameError:

intelmq/bots/outputs/smtp_batch/output.py

@@ -123,7 +123,7 @@ def cli_run(self):
         with open(self.mail_template) as f:
             self.mail_contents = f.read()
         if self.alternative_mails:
-            with open(self.alternative_mails, "r") as f:
+            with open(self.alternative_mails) as f:
                 self.alternative_mail = {row[0]: row[1] for row in csv.reader(f, delimiter=",")}
 
         print("Preparing mail queue...")

intelmq/tests/lib/test_pipeline.py

@@ -266,8 +266,8 @@ def test_reject(self):
         self.pipe.reject_message()
         self.assertEqual(SAMPLES['normal'][1], self.pipe.receive())
 
-    @unittest.skipIf(os.getenv('CI') == 'true' and sys.version_info[:2] == (3, 8),
-                     'Fails on CI with Python 3.8')
+    @unittest.skipIf(os.getenv('CI') == 'true' and sys.version_info[:2] in ((3, 8), (3, 11)),
+                     'Fails on CI with Python 3.8 and 3.11')
     def test_acknowledge(self):
         self.pipe.send(SAMPLES['normal'][0])
         self.pipe.receive()

setup.py

@@ -17,7 +17,6 @@
     'redis>=2.10',
     'requests>=2.2.0',
     'ruamel.yaml',
-    'importlib-metadata; python_version < "3.8"'
 ]
 
 TESTS_REQUIRES = [
@@ -56,7 +55,7 @@
     version=__version__,  # noqa: F821
     maintainer='Sebastian Wagner',
     maintainer_email='[email protected]',
-    python_requires='>=3.7',
+    python_requires='>=3.8',
     install_requires=REQUIRES,
     tests_require=TESTS_REQUIRES,
     test_suite='intelmq.tests',
@@ -85,11 +84,11 @@
         'Operating System :: POSIX :: Linux',
         'Programming Language :: Python',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.7',
-        'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',
         'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
+        'Programming Language :: Python :: 3.13',
         'Programming Language :: Python :: 3 :: Only',
         'Programming Language :: Python :: Implementation :: CPython',
         'Topic :: Security',