Merge pull request #2542 from sebix/fix-tests

Fixes to the tests: pymssql, psql, user & group

.github/workflows/python/github.py

@@ -31,20 +31,20 @@ def __init__(self):
 
     def get_reviews(self):
         """ Get a list of reviews on a Github pull request as json object """
-        reviews = self.session.get(self.api + 'repos/{}/pulls/{}/reviews'.format(self.github_repository, self.pr_id))
+        reviews = self.session.get(self.api + f'repos/{self.github_repository}/pulls/{self.pr_id}/reviews')
         reviews.raise_for_status()
         return reviews.json()
 
     def update_review(self, review_id, body):
         """ Update a review given by `review_id` and set its body to `body` """
         payload = {'body': body}
-        resp = self.session.put(self.api + 'repos/{}/pulls/{}/reviews/{}'.format(self.github_repository, self.pr_id, review_id), json=payload)
+        resp = self.session.put(self.api + f'repos/{self.github_repository}/pulls/{self.pr_id}/reviews/{review_id}', json=payload)
         resp.raise_for_status()
         return resp.json()
 
     def post_review(self, body):
         """ Post a pull request review containing `body` and requesting changes """
         payload = {'body': body, 'event': "REQUEST_CHANGES"}
-        resp = self.session.post(self.api + 'repos/{}/pulls/{}/reviews'.format(self.github_repository, self.pr_id), json=payload)
+        resp = self.session.post(self.api + f'repos/{self.github_repository}/pulls/{self.pr_id}/reviews', json=payload)
         resp.raise_for_status()
         return resp.json()

.github/workflows/python/pycodestyle_comment.py

@@ -34,7 +34,7 @@ def style_error_format(style_error_list) -> str:
     """ Format the list of pycodestyle errors and return them a one string. """
     ret = ''
     for error in style_error_list:
-        ret += '* {}\n'.format(error)
+        ret += f'* {error}\n'
     return ret
 
 
@@ -45,7 +45,7 @@ def style_error_format(style_error_list) -> str:
         style_errors = list_style_errors()
 
         if style_errors:
-            print("Found {} errors.".format(len(style_errors)))
+            print(f"Found {len(style_errors)} errors.")
 
         gh = github.Github()
 

.github/workflows/scripts/setup-full.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# SPDX-FileCopyrightText: 2020 Birger Schacht
+# SPDX-FileCopyrightText: 2020 Birger Schacht, 2024 Institute for Common Good Technology
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 set -x
@@ -14,6 +14,14 @@ echo -e '-XX:+DisableExplicitGC\n-Djdk.io.permissionsUseCanonicalPath=true\n-Dlo
 sudo chown -R elasticsearch:elasticsearch /etc/default/elasticsearch
 sudo systemctl start elasticsearch
 
+sudo apt update
+if [ $python_version == '3.8' ]; then
+	# for pymssql there are no wheels for 3.8 https://github.com/certtools/intelmq/issues/2539
+	DEBIAN_FRONTEND="noninteractive" sudo -E apt install -y build-essential freetds-dev libssl-dev libkrb5-dev
+fi
+# for psql (used below)
+DEBIAN_FRONTEND="noninteractive" sudo -E apt install -y postgresql-client-14
+
 # Install the dependencies of all the bots
 pip install wheel
 for file in intelmq/bots/*/*/REQUIREMENTS.txt; do
@@ -30,7 +38,16 @@ done
 # Setup sudo and install intelmq
 sudo sed -i '/^Defaults\tsecure_path.*$/ d' /etc/sudoers
 sudo pip install .
-sudo intelmqsetup --skip-ownership
+
+intelmq_user_exists=$(getent passwd intelmq ||:)
+if [[ "$UID" -eq '0' && -z "$intelmq_user_exists" ]]; then
+	# create an unprivileged user, if currently running as root. Otherwise dropping privileges won't work
+	groupadd -r intelmq
+	useradd -r -d /var/lib/intelmq/ -c "user running intelmq" -g intelmq -s /bin/bash intelmq
+	sudo intelmqsetup
+else
+	sudo intelmqsetup --skip-ownership
+fi
 
 # Initialize the postgres database
 intelmq_psql_initdb

.github/workflows/unittests.yml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
         type: ['full', 'basic']
 
     services:
@@ -59,6 +59,7 @@ jobs:
         PGPORT: 5432
         PGUSER: intelmq
         PGPASSWORD: intelmq
+        python_version: ${{ matrix.python-version }}
       run: bash .github/workflows/scripts/setup-full.sh
 
     - name: Install test dependencies

CHANGELOG.md

@@ -12,6 +12,7 @@
 ### Configuration
 
 ### Core
+- Python 3.8 or newer is required (PR#2541 by Sebastian Wagner).
 
 ### Development
 
@@ -31,12 +32,20 @@
 #### Experts
 
 #### Outputs
+- `intelmq.bots.outputs.cif3.output`:
+  - The requirement can only be installed on Python version < 3.12.
+  - Add a check on the Python version and exit if incompatible.
+  - Add a deprecation warning (PR#2544 by Sebastian Wagner)
 
 ### Documentation
 
 ### Packaging
 
 ### Tests
+- Install build dependencies for `pymssql` on Python 3.8 as there are no wheels available for this Python version (PR#2542 by Sebastian Wagner).
+- Install `psql` explicitly for workflow support on other platforms such as act (PR#2542 by Sebastian Wagner).
+- Create intelmq user & group if running privileged to allow dropping privileges (PR#2542 by Sebastian Wagner).
+- `intelmq.tests.lib.test_pipeline.TestAmqp.test_acknowledge`: Also skip on Python 3.11 besides on 3.8 when running on CI (PR#2542 by Sebastian Wagner).
 
 ### Tools
 

NEWS.md

@@ -14,6 +14,12 @@ Please refer to the change log for a full list of changes.
 ---------------------------------
 
 ### Requirements
+Python 3.8 or newer is required.
+
+## Bots
+#### CIF 3 API
+The CIF 3 API Output bot is not compatible with Python version greater or equal to 3.12 and will be removed in the future due to lack of maintenance.
+See https://lists.cert.at/pipermail/intelmq-users/2024-December/000474.html for more information.
 
 ### Tools
 
@@ -36,7 +42,7 @@ No changes are required by administrators.
 ----------------------------------
 
 ### Documentation
-The documentation is now available at [docs.intelmq.org](https://docs.intelmq.org/). Documentation has been updated and restructured into User, Administrator and Developer Guide. It provides modern look with various quality of life improvements. Big thanks to to @gethvi. 
+The documentation is now available at [docs.intelmq.org](https://docs.intelmq.org/). Documentation has been updated and restructured into User, Administrator and Developer Guide. It provides modern look with various quality of life improvements. Big thanks to to @gethvi.
 We now have a slick, modern mkdocs based documentation. Please do check it out!
 
 
@@ -55,7 +61,7 @@ Shadowserver adds new scans on a nearly weekly basis. IntelMQ's release cycle an
 We therefore (thanks to @eslif2) move the shadowserver reports collector and parser to a new, dynamic system. It can:
 
  - fetch the shadowserver schema from shadowserver (https://interchange.shadowserver.org/intelmq/v1/schema)
- - dynamically collect new reports (see also https://docs.intelmq.org/latest/user/bots/?h=shadow#shadowserver-reports-api) 
+ - dynamically collect new reports (see also https://docs.intelmq.org/latest/user/bots/?h=shadow#shadowserver-reports-api)
  - parse the new reports
 
 **Note well**: if your IntelMQ system runs in an airgapped environment or if it may only reach out to specific IPs/sites, you should read the notes here:
@@ -86,7 +92,7 @@ Quite a few changes (thanks to Kamil, @gethvi) on AMQP
 ### General changes and bug fixes
 
 Digital Trust Center fixed a bug where the config was loaded twice in intelmqctl which created quite some speedups. Thanks!
-This speeds up IntelMQ API calls. 
+This speeds up IntelMQ API calls.
 
 ### Data Format
 

contrib/example-extension-package/setup.py

@@ -28,7 +28,7 @@
     entry_point = '.'.join(file.with_suffix('').parts)
     file = Path(str(file).replace('intelmq/bots', 'mybots/bots'))
     module = '.'.join(file.with_suffix('').parts)
-    BOTS.append('{0} = {1}:BOT.run'.format(entry_point, module))
+    BOTS.append(f'{entry_point} = {module}:BOT.run')
 
 setup(
     name='intelmq-example-extension',

intelmq/bots/collectors/rsync/collector_rsync.py

@@ -13,7 +13,7 @@
 from intelmq.lib.bot import CollectorBot
 
 
-class Time(object):
+class Time:
     def __init__(self, delta=None):
         """ Delta is a datetime.timedelta JSON string, ex: '{days=-1}'. """
         self.time = datetime.now()

intelmq/bots/experts/modify/expert.py

@@ -17,10 +17,7 @@ def is_re_pattern(value):
     """
     Checks if the given value is a re compiled pattern
     """
-    if sys.version_info > (3, 7):
-        return isinstance(value, re.Pattern)
-    else:
-        return hasattr(value, "pattern")
+    return isinstance(value, re.Pattern)
 
 
 class MatchGroupMapping:

intelmq/bots/outputs/cif3/REQUIREMENTS.txt

@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# SPDX-FileCopyrightText: 2022 REN-ISAC
+# SPDX-FileCopyrightText: 2022 REN-ISAC, 2024 Institute for Common Good Technology
 
-cifsdk>=3.0.0rc4,<4.0
+cifsdk>=3.0.0rc4,<4.0; python_version < '3.12'

intelmq/bots/outputs/cif3/output.py

@@ -32,6 +32,7 @@
     JsonLib = None
 
 from datetime import datetime
+from sys import version_info
 from typing import Optional, List
 
 from intelmq.lib.bot import OutputBot
@@ -91,6 +92,9 @@ class CIF3OutputBot(OutputBot):
     _is_multithreadable = False
 
     def init(self):
+        raise DeprecationWarning("The CIFv3 API is deprecated and will be remove in IntelMQ version 4.0. See https://lists.cert.at/pipermail/intelmq-users/2024-December/000474.html")
+        if version_info >= (3, 12):
+            raise ValueError("This bot is not compatible with Python >= 3.12. See https://lists.cert.at/pipermail/intelmq-users/2024-December/000474.html")
         try:
             cifsdk_version = int(get_cifsdk_version().get('version').split('.')[0])
         except NameError:

intelmq/bots/outputs/smtp_batch/output.py

@@ -123,7 +123,7 @@ def cli_run(self):
         with open(self.mail_template) as f:
             self.mail_contents = f.read()
         if self.alternative_mails:
-            with open(self.alternative_mails, "r") as f:
+            with open(self.alternative_mails) as f:
                 self.alternative_mail = {row[0]: row[1] for row in csv.reader(f, delimiter=",")}
 
         print("Preparing mail queue...")

intelmq/tests/lib/test_pipeline.py

@@ -266,8 +266,8 @@ def test_reject(self):
         self.pipe.reject_message()
         self.assertEqual(SAMPLES['normal'][1], self.pipe.receive())
 
-    @unittest.skipIf(os.getenv('CI') == 'true' and sys.version_info[:2] == (3, 8),
-                     'Fails on CI with Python 3.8')
+    @unittest.skipIf(os.getenv('CI') == 'true' and sys.version_info[:2] in ((3, 8), (3, 11)),
+                     'Fails on CI with Python 3.8 and 3.11')
     def test_acknowledge(self):
         self.pipe.send(SAMPLES['normal'][0])
         self.pipe.receive()

setup.py

@@ -17,7 +17,6 @@
     'redis>=2.10',
     'requests>=2.2.0',
     'ruamel.yaml',
-    'importlib-metadata; python_version < "3.8"'
 ]
 
 TESTS_REQUIRES = [
@@ -56,7 +55,7 @@
     version=__version__,  # noqa: F821
     maintainer='Sebastian Wagner',
     maintainer_email='[email protected]',
-    python_requires='>=3.7',
+    python_requires='>=3.8',
     install_requires=REQUIRES,
     tests_require=TESTS_REQUIRES,
     test_suite='intelmq.tests',
@@ -85,11 +84,11 @@
         'Operating System :: POSIX :: Linux',
         'Programming Language :: Python',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.7',
-        'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',
         'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
+        'Programming Language :: Python :: 3.13',
         'Programming Language :: Python :: 3 :: Only',
         'Programming Language :: Python :: Implementation :: CPython',
         'Topic :: Security',