diff --git a/.github/workflows/python/github.py b/.github/workflows/python/github.py index 89e72b5991..1abbbb03c9 100644 --- a/.github/workflows/python/github.py +++ b/.github/workflows/python/github.py @@ -31,20 +31,20 @@ def __init__(self): def get_reviews(self): """ Get a list of reviews on a Github pull request as json object """ - reviews = self.session.get(self.api + 'repos/{}/pulls/{}/reviews'.format(self.github_repository, self.pr_id)) + reviews = self.session.get(self.api + f'repos/{self.github_repository}/pulls/{self.pr_id}/reviews') reviews.raise_for_status() return reviews.json() def update_review(self, review_id, body): """ Update a review given by `review_id` and set its body to `body` """ payload = {'body': body} - resp = self.session.put(self.api + 'repos/{}/pulls/{}/reviews/{}'.format(self.github_repository, self.pr_id, review_id), json=payload) + resp = self.session.put(self.api + f'repos/{self.github_repository}/pulls/{self.pr_id}/reviews/{review_id}', json=payload) resp.raise_for_status() return resp.json() def post_review(self, body): """ Post a pull request review containing `body` and requesting changes """ payload = {'body': body, 'event': "REQUEST_CHANGES"} - resp = self.session.post(self.api + 'repos/{}/pulls/{}/reviews'.format(self.github_repository, self.pr_id), json=payload) + resp = self.session.post(self.api + f'repos/{self.github_repository}/pulls/{self.pr_id}/reviews', json=payload) resp.raise_for_status() return resp.json() diff --git a/.github/workflows/python/pycodestyle_comment.py b/.github/workflows/python/pycodestyle_comment.py index 89eca1936f..4cce4d4028 100644 --- a/.github/workflows/python/pycodestyle_comment.py +++ b/.github/workflows/python/pycodestyle_comment.py @@ -34,7 +34,7 @@ def style_error_format(style_error_list) -> str: """ Format the list of pycodestyle errors and return them a one string. """ ret = '' for error in style_error_list: - ret += '* {}\n'.format(error) + ret += f'* {error}\n' return ret @@ -45,7 +45,7 @@ def style_error_format(style_error_list) -> str: style_errors = list_style_errors() if style_errors: - print("Found {} errors.".format(len(style_errors))) + print(f"Found {len(style_errors)} errors.") gh = github.Github() diff --git a/.github/workflows/scripts/setup-full.sh b/.github/workflows/scripts/setup-full.sh index 63e9353952..e546c0e459 100644 --- a/.github/workflows/scripts/setup-full.sh +++ b/.github/workflows/scripts/setup-full.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# SPDX-FileCopyrightText: 2020 Birger Schacht +# SPDX-FileCopyrightText: 2020 Birger Schacht, 2024 Institute for Common Good Technology # SPDX-License-Identifier: AGPL-3.0-or-later set -x @@ -14,6 +14,14 @@ echo -e '-XX:+DisableExplicitGC\n-Djdk.io.permissionsUseCanonicalPath=true\n-Dlo sudo chown -R elasticsearch:elasticsearch /etc/default/elasticsearch sudo systemctl start elasticsearch +sudo apt update +if [ $python_version == '3.8' ]; then + # for pymssql there are no wheels for 3.8 https://github.com/certtools/intelmq/issues/2539 + DEBIAN_FRONTEND="noninteractive" sudo -E apt install -y build-essential freetds-dev libssl-dev libkrb5-dev +fi +# for psql (used below) +DEBIAN_FRONTEND="noninteractive" sudo -E apt install -y postgresql-client-14 + # Install the dependencies of all the bots pip install wheel for file in intelmq/bots/*/*/REQUIREMENTS.txt; do @@ -30,7 +38,16 @@ done # Setup sudo and install intelmq sudo sed -i '/^Defaults\tsecure_path.*$/ d' /etc/sudoers sudo pip install . -sudo intelmqsetup --skip-ownership + +intelmq_user_exists=$(getent passwd intelmq ||:) +if [[ "$UID" -eq '0' && -z "$intelmq_user_exists" ]]; then + # create an unprivileged user, if currently running as root. Otherwise dropping privileges won't work + groupadd -r intelmq + useradd -r -d /var/lib/intelmq/ -c "user running intelmq" -g intelmq -s /bin/bash intelmq + sudo intelmqsetup +else + sudo intelmqsetup --skip-ownership +fi # Initialize the postgres database intelmq_psql_initdb diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 1dee5735f5..4325e96b1c 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] type: ['full', 'basic'] services: @@ -59,6 +59,7 @@ jobs: PGPORT: 5432 PGUSER: intelmq PGPASSWORD: intelmq + python_version: ${{ matrix.python-version }} run: bash .github/workflows/scripts/setup-full.sh - name: Install test dependencies diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b31d70355..7b82d0add0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ ### Configuration ### Core +- Python 3.8 or newer is required (PR#2541 by Sebastian Wagner). ### Development @@ -19,6 +20,8 @@ ### Bots #### Collectors +- `intelmq.bots.collectors.shadowserver.collector_reports_api.py`: + - Fixed behaviour if parameter `types` value is empty string, behave the same way as not set, not like no type. #### Parsers - `intelmq.bots.parsers.shadowserver._config`: @@ -29,12 +32,20 @@ #### Experts #### Outputs +- `intelmq.bots.outputs.cif3.output`: + - The requirement can only be installed on Python version < 3.12. + - Add a check on the Python version and exit if incompatible. + - Add a deprecation warning (PR#2544 by Sebastian Wagner) ### Documentation ### Packaging ### Tests +- Install build dependencies for `pymssql` on Python 3.8 as there are no wheels available for this Python version (PR#2542 by Sebastian Wagner). +- Install `psql` explicitly for workflow support on other platforms such as act (PR#2542 by Sebastian Wagner). +- Create intelmq user & group if running privileged to allow dropping privileges (PR#2542 by Sebastian Wagner). +- `intelmq.tests.lib.test_pipeline.TestAmqp.test_acknowledge`: Also skip on Python 3.11 besides on 3.8 when running on CI (PR#2542 by Sebastian Wagner). ### Tools diff --git a/NEWS.md b/NEWS.md index 6f79b6ea7d..a3c242979a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -14,6 +14,12 @@ Please refer to the change log for a full list of changes. --------------------------------- ### Requirements +Python 3.8 or newer is required. + +## Bots +#### CIF 3 API +The CIF 3 API Output bot is not compatible with Python version greater or equal to 3.12 and will be removed in the future due to lack of maintenance. +See https://lists.cert.at/pipermail/intelmq-users/2024-December/000474.html for more information. ### Tools @@ -36,7 +42,7 @@ No changes are required by administrators. ---------------------------------- ### Documentation -The documentation is now available at [docs.intelmq.org](https://docs.intelmq.org/). Documentation has been updated and restructured into User, Administrator and Developer Guide. It provides modern look with various quality of life improvements. Big thanks to to @gethvi. +The documentation is now available at [docs.intelmq.org](https://docs.intelmq.org/). Documentation has been updated and restructured into User, Administrator and Developer Guide. It provides modern look with various quality of life improvements. Big thanks to to @gethvi. We now have a slick, modern mkdocs based documentation. Please do check it out! @@ -55,7 +61,7 @@ Shadowserver adds new scans on a nearly weekly basis. IntelMQ's release cycle an We therefore (thanks to @eslif2) move the shadowserver reports collector and parser to a new, dynamic system. It can: - fetch the shadowserver schema from shadowserver (https://interchange.shadowserver.org/intelmq/v1/schema) - - dynamically collect new reports (see also https://docs.intelmq.org/latest/user/bots/?h=shadow#shadowserver-reports-api) + - dynamically collect new reports (see also https://docs.intelmq.org/latest/user/bots/?h=shadow#shadowserver-reports-api) - parse the new reports **Note well**: if your IntelMQ system runs in an airgapped environment or if it may only reach out to specific IPs/sites, you should read the notes here: @@ -86,7 +92,7 @@ Quite a few changes (thanks to Kamil, @gethvi) on AMQP ### General changes and bug fixes Digital Trust Center fixed a bug where the config was loaded twice in intelmqctl which created quite some speedups. Thanks! -This speeds up IntelMQ API calls. +This speeds up IntelMQ API calls. ### Data Format diff --git a/contrib/example-extension-package/setup.py b/contrib/example-extension-package/setup.py index 3d6774693f..d662eca34d 100644 --- a/contrib/example-extension-package/setup.py +++ b/contrib/example-extension-package/setup.py @@ -28,7 +28,7 @@ entry_point = '.'.join(file.with_suffix('').parts) file = Path(str(file).replace('intelmq/bots', 'mybots/bots')) module = '.'.join(file.with_suffix('').parts) - BOTS.append('{0} = {1}:BOT.run'.format(entry_point, module)) + BOTS.append(f'{entry_point} = {module}:BOT.run') setup( name='intelmq-example-extension', diff --git a/intelmq/bots/collectors/rsync/collector_rsync.py b/intelmq/bots/collectors/rsync/collector_rsync.py index 112deb1f17..3829c8a3f8 100644 --- a/intelmq/bots/collectors/rsync/collector_rsync.py +++ b/intelmq/bots/collectors/rsync/collector_rsync.py @@ -13,7 +13,7 @@ from intelmq.lib.bot import CollectorBot -class Time(object): +class Time: def __init__(self, delta=None): """ Delta is a datetime.timedelta JSON string, ex: '{days=-1}'. """ self.time = datetime.now() diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py index 3fc3d2f9f6..2fa11a3c2b 100644 --- a/intelmq/bots/collectors/shadowserver/collector_reports_api.py +++ b/intelmq/bots/collectors/shadowserver/collector_reports_api.py @@ -33,7 +33,7 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin): reports (list): A list of strings or a comma-separated list of the mailing lists you want to process. types (list): - A list of strings or a string of comma-separated values with the names of reporttypes you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined'). + A list of strings or a string of comma-separated values with the names of report types you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined'). """ country = None @@ -48,6 +48,7 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin): redis_cache_ttl: int = 864000 # 10 days redis_cache_password: Optional[str] = None _report_list = [] + _type_list = [] def init(self): if not self.api_key: @@ -62,7 +63,11 @@ def init(self): elif isinstance(self.reports, list): self._report_list = self.reports if isinstance(self.types, str): - self.types = self.types.split(',') + # if types is an empty string (or only contains whitespace), behave as if the parameter is not set and select all types + types = self.types.strip() + self._type_list = types.split(',') if types else [] + elif isinstance(self.types, list): + self._type_list = self.types if self.country and self.country not in self._report_list: self.logger.warn("Deprecated parameter 'country' found. Please use 'reports' instead. The backwards-compatibility will be removed in IntelMQ version 4.0.0.") self._report_list.append(self.country) @@ -111,8 +116,8 @@ def _reports_list(self, date=None): self.logger.debug('There was an error downloading the reports: %s', reports['error']) return None - if self.types: - reports = [report for report in reports if any(report['type'] == rtype for rtype in self.types)] + if self._type_list: + reports = [report for report in reports if any(report['type'] == rtype for rtype in self._type_list)] return reports def _report_download(self, reportid: str): diff --git a/intelmq/bots/experts/modify/expert.py b/intelmq/bots/experts/modify/expert.py index a9945449a4..f6cdbbb439 100644 --- a/intelmq/bots/experts/modify/expert.py +++ b/intelmq/bots/experts/modify/expert.py @@ -17,10 +17,7 @@ def is_re_pattern(value): """ Checks if the given value is a re compiled pattern """ - if sys.version_info > (3, 7): - return isinstance(value, re.Pattern) - else: - return hasattr(value, "pattern") + return isinstance(value, re.Pattern) class MatchGroupMapping: diff --git a/intelmq/bots/outputs/cif3/REQUIREMENTS.txt b/intelmq/bots/outputs/cif3/REQUIREMENTS.txt index 9b898b618a..0a3bef49d9 100644 --- a/intelmq/bots/outputs/cif3/REQUIREMENTS.txt +++ b/intelmq/bots/outputs/cif3/REQUIREMENTS.txt @@ -1,4 +1,4 @@ # SPDX-License-Identifier: AGPL-3.0-or-later -# SPDX-FileCopyrightText: 2022 REN-ISAC +# SPDX-FileCopyrightText: 2022 REN-ISAC, 2024 Institute for Common Good Technology -cifsdk>=3.0.0rc4,<4.0 +cifsdk>=3.0.0rc4,<4.0; python_version < '3.12' diff --git a/intelmq/bots/outputs/cif3/output.py b/intelmq/bots/outputs/cif3/output.py index 537ac40b13..f7682c4107 100644 --- a/intelmq/bots/outputs/cif3/output.py +++ b/intelmq/bots/outputs/cif3/output.py @@ -32,6 +32,7 @@ JsonLib = None from datetime import datetime +from sys import version_info from typing import Optional, List from intelmq.lib.bot import OutputBot @@ -91,6 +92,9 @@ class CIF3OutputBot(OutputBot): _is_multithreadable = False def init(self): + raise DeprecationWarning("The CIFv3 API is deprecated and will be remove in IntelMQ version 4.0. See https://lists.cert.at/pipermail/intelmq-users/2024-December/000474.html") + if version_info >= (3, 12): + raise ValueError("This bot is not compatible with Python >= 3.12. See https://lists.cert.at/pipermail/intelmq-users/2024-December/000474.html") try: cifsdk_version = int(get_cifsdk_version().get('version').split('.')[0]) except NameError: diff --git a/intelmq/bots/outputs/smtp_batch/output.py b/intelmq/bots/outputs/smtp_batch/output.py index d24d114671..5661ea7339 100644 --- a/intelmq/bots/outputs/smtp_batch/output.py +++ b/intelmq/bots/outputs/smtp_batch/output.py @@ -123,7 +123,7 @@ def cli_run(self): with open(self.mail_template) as f: self.mail_contents = f.read() if self.alternative_mails: - with open(self.alternative_mails, "r") as f: + with open(self.alternative_mails) as f: self.alternative_mail = {row[0]: row[1] for row in csv.reader(f, delimiter=",")} print("Preparing mail queue...") diff --git a/intelmq/tests/lib/test_pipeline.py b/intelmq/tests/lib/test_pipeline.py index 39f75eb0fb..51c460668c 100644 --- a/intelmq/tests/lib/test_pipeline.py +++ b/intelmq/tests/lib/test_pipeline.py @@ -266,8 +266,8 @@ def test_reject(self): self.pipe.reject_message() self.assertEqual(SAMPLES['normal'][1], self.pipe.receive()) - @unittest.skipIf(os.getenv('CI') == 'true' and sys.version_info[:2] == (3, 8), - 'Fails on CI with Python 3.8') + @unittest.skipIf(os.getenv('CI') == 'true' and sys.version_info[:2] in ((3, 8), (3, 11)), + 'Fails on CI with Python 3.8 and 3.11') def test_acknowledge(self): self.pipe.send(SAMPLES['normal'][0]) self.pipe.receive() diff --git a/setup.py b/setup.py index 7bbe279fc2..55725c37e2 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,6 @@ 'redis>=2.10', 'requests>=2.2.0', 'ruamel.yaml', - 'importlib-metadata; python_version < "3.8"' ] TESTS_REQUIRES = [ @@ -56,7 +55,7 @@ version=__version__, # noqa: F821 maintainer='Sebastian Wagner', maintainer_email='intelmq-dev@lists.cert.at', - python_requires='>=3.7', + python_requires='>=3.8', install_requires=REQUIRES, tests_require=TESTS_REQUIRES, test_suite='intelmq.tests', @@ -85,11 +84,11 @@ 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: Implementation :: CPython', 'Topic :: Security',