update CI and linting rules

.github/workflows/tests.yml

@@ -4,26 +4,35 @@ name: Tests
 on:
   - pull_request
   - push
+  - release
+  - workflow_dispatch
+
+# cancel the current workflow if another commit was pushed on the same PR or reference
+# uses the GitHub workflow name to avoid collision with other workflows running on the same PR/reference
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 jobs:
   # see: https://github.com/fkirc/skip-duplicate-actions
   skip_duplicate:
     continue-on-error: true
     runs-on: ubuntu-latest
     outputs:
-      should_skip: ${{ steps.skip_check.outputs.should_skip && ! contains(github.ref, 'refs/tags') && ! contains(github.ref, 'refs/heads/master') }}
+      should_skip: ${{ steps.skip_check.outputs.should_skip }}
     steps:
       - id: skip_check
         uses: fkirc/skip-duplicate-actions@master
         with:
-          concurrent_skipping: "same_content"
+          concurrent_skipping: "same_content_newer"
           skip_after_successful_duplicate: "true"
+          cancel_others: "true"
           do_not_skip: '["workflow_dispatch", "schedule", "release"]'
 
   # see: https://github.com/actions/setup-python
   tests:
     needs: skip_duplicate
-    if: ${{ needs.skip_duplicate.outputs.should_skip != 'true' }}
+    if: ${{ needs.skip_duplicate.outputs.should_skip != 'true' && ! contains(github.ref, 'refs/tags') && ! contains(github.ref, 'refs/heads/master') }}
     runs-on: ${{ matrix.os }}
     continue-on-error: ${{ matrix.allow-failure }}
     env:
@@ -59,7 +68,7 @@ jobs:
             # FIXME:
             #   Not using 3.11 because of problems with pylint false-positives about missing builtin references.
             #   https://github.com/PyCQA/pylint/issues/6535
-            python-version: "3.10"
+            python-version: "3.12"
             allow-failure: false
             test-case: check
           # remote test (note that network tests are skipped since the remote server won't have network mode enabled)
@@ -85,11 +94,11 @@ jobs:
             allow-failure: true
             test-case: test-docker
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v6
         with:
           fetch-depth: "0"
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v6
         if: ${{ matrix.python-version != 'none' }}
         with:
           python-version: ${{ matrix.python-version }}

.pylintrc

@@ -28,7 +28,8 @@ limit-inference-results=100
 
 # List of plugins (as comma separated values of python module names) to load,
 # usually to register additional checkers.
-load-plugins=
+load-plugins=pylint_per_file_ignores,
+             pylint_quotes
 
 # Pickle collected data for later comparisons.
 persistent=yes
@@ -88,6 +89,7 @@ disable=C0111,missing-docstring,
         R1721,unnecessary-comprehension,
         R1725,super-with-arguments,
         R1729,use-a-generator,
+        R1735,use-dict-literal,
         W0108,unnecessary-lambda,
         W0143,comparison-with-callable,
         W0232,no-init,
@@ -105,6 +107,10 @@ disable=C0111,missing-docstring,
 #   C0411,wrong-import-order
 #   C0412,ungrouped-imports
 
+per-file-ignores =
+    magpie/alembic/versions/*:C0103,
+    tests/*:R0917,R1729,W0719
+
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
 # multiple time (only on the command line, not in the configuration file where
@@ -544,6 +550,8 @@ preferred-modules=
 # Maximum number of arguments for function / method.
 max-args=20
 
+max-positional-arguments=10
+
 # Maximum number of attributes for a class (see R0902).
 max-attributes=20
 
@@ -576,5 +584,5 @@ min-public-methods=0
 
 # Exceptions that will emit a warning when being caught. Defaults to
 # "BaseException, Exception".
-overgeneral-exceptions=BaseException,
-                       Exception
+overgeneral-exceptions=builtins.BaseException,
+                       builtins.Exception

Makefile

@@ -528,7 +528,6 @@ check-lint-only: mkdir-reports		## run linting code style checks
 	@-rm -fr "$(REPORTS_DIR)/check-lint.txt"
 	@bash -c '$(CONDA_CMD) \
 		pylint \
-			--load-plugins pylint_quotes \
 			--rcfile="$(APP_ROOT)/.pylintrc" \
 			--reports y \
 			"$(APP_ROOT)/$(APP_NAME)" "$(APP_ROOT)/$(APP_NAME)/alembic" "$(APP_ROOT)/docs" "$(APP_ROOT)/tests" \

magpie/adapter/__init__.py

@@ -248,7 +248,7 @@ def owsregistry_factory(self, request):
         .. versionadded:: 3.18
             Available only in ``Twitcher >= 0.6.x``.
         """
-        return OWSRegistry(self.servicestore_factory(request))
+        return OWSRegistry(self.servicestore_factory(request))  # pylint: disable=E0606
 
     def owssecurity_factory(self, request=None):  # noqa  # pylint: disable=W0221  # diff between Twitcher 0.5.x/0.6.x
         # type: (Optional[AnySettingsContainer]) -> MagpieOWSSecurity

magpie/alembic/versions/2023-11-10_91af68bcdc67_case_insensitive_user_name_constraint.py

@@ -36,7 +36,7 @@ def upgrade():
             unique=True,
         )
     except sa.exc.IntegrityError as e:
-        raise Exception("{}\nPlease manually update conflicting user_names and try again".format(e)) from e
+        raise RuntimeError("{}\nPlease manually update conflicting user_names and try again".format(e)) from e
     session = Session(bind=op.get_bind())
     for user in session.execute(sa.select([users])):
         lower_user_name = user.user_name.lower()

magpie/api/exception.py

@@ -58,7 +58,7 @@
     Pattern = type(re.compile(""))
 
 
-def verify_param(  # noqa: E126  # pylint: disable=R0913,too-many-arguments
+def verify_param(  # noqa: E126  # pylint: disable=R0913,R0917
                  # --- verification values ---      # noqa: E126
                  param,                             # type: Any
                  param_compare=None,                # type: Optional[Union[Any, List[Any]]]

magpie/api/management/resource/resource_views.py

@@ -335,7 +335,7 @@ def update_permissions(request):
             resource_full_type += "/" + resource_type
         if permission:
             cfg_entry = {
-                "service": service_name,  # noqa
+                "service": service_name,  # noqa  # pylint: disable=E0606  # set on 1st iteration, service must be 1st
                 "resource": resource_full_path,
                 "type": resource_type if resource_type == "service" else resource_full_type,
                 "permission": permission,

magpie/cli/batch_update_users.py

@@ -63,7 +63,7 @@ def create_users(user_config, magpie_url, magpie_admin_username, magpie_admin_pa
     password_length = password_length or get_constant("MAGPIE_PASSWORD_MIN_LENGTH")
     for usr_cfg in user_config:
         if not usr_cfg.get("password"):
-            LOGGER.warning("No password provided for user: '%s'. Will auto-generate random value.")
+            LOGGER.warning("No password provided for user: '%s'. Will auto-generate random value.", usr_cfg["username"])
             usr_cfg["password"] = pseudo_random_string(length=password_length)
         data = {"user_name": usr_cfg["username"], "password": usr_cfg["password"],
                 "group_name": usr_cfg["group"], "email": usr_cfg["email"]}

magpie/compat.py

@@ -3,7 +3,7 @@
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from typing import Tuple, Union
+    from typing import Union
 
 
 class VersionInterface(object):
@@ -27,19 +27,19 @@ def patch(self):
 
 
 try:
+    from typing import NamedTuple, Tuple
+
     from packaging.version import InvalidVersion  # pylint: disable=unused-import
     from packaging.version import Version as BaseVersion  # pylint: disable=unused-import
-    from typing import NamedTuple, Tuple
 
     class TupleVersion(NamedTuple):
         epoch: int
-        release: tuple[int, ...]
-        dev: tuple[str, int] | None
-        pre: tuple[str, int] | None
-        post: tuple[str, int] | None
+        release: Tuple[int, ...]
+        dev: Tuple[str, int] | None
+        pre: Tuple[str, int] | None
+        post: Tuple[str, int] | None
         local: Tuple[int | str, ...] | None
 
-
     class LooseVersion(BaseVersion, VersionInterface):
         # override '_version' explicitly with the equivalent procedure of previous versions
         # this avoids attribute errors and unnecessary warnings

magpie/config.py

@@ -73,8 +73,7 @@ def set_defaults(validator, properties, instance, schema):
         for prop, child_schema in properties.items():
             if "default" in child_schema:
                 instance.setdefault(prop, child_schema["default"])
-        for error in validate_properties(validator, properties, instance, schema):
-            yield error
+        yield from validate_properties(validator, properties, instance, schema)
 
     return validators.extend(validator_class, {"properties": set_defaults})
 

magpie/models.py

@@ -379,7 +379,7 @@ def get(cls,            # pylint: disable=W0237,arguments-renamed
         for _status in status:
             _status = cls._get_one(_status)
             if combined is not None and _status is not None:
-                combined = (combined | _status)
+                combined = combined | _status
             else:
                 combined = combined or _status
         return UserStatuses(combined)

magpie/typedefs.py

@@ -58,7 +58,7 @@
     _JsonObjectItem = Dict[str, Union[AnyValue, _JSON, _JsonObjectItemAlias, _JsonListItemAlias]]
     _JsonListItem = List[Union[AnyValue, _JSON, _JsonObjectItem, _JsonListItemAlias]]
     _JsonItem = Union[AnyValue, _JSON, _JsonObjectItem, _JsonListItem]
-    JSON = Union[Dict[str, Union[_JSON, _JsonItem]], List[Union[_JSON, _JsonItem]], AnyValue]
+    JSON = Union[Dict[str, Union[_JSON, _JsonItem]], List[Union[_JSON, _JsonItem]], AnyValue]  # pylint: disable=C0103
 
     GroupPriority = Union[int, Type[math.inf]]
     UserServicesType = Union[Dict[Str, Dict[Str, Any]], List[Dict[Str, Any]]]

magpie/utils.py

@@ -178,7 +178,7 @@ def raise_log(msg, exception=Exception, logger=None, level=logging.ERROR):
     logger.log(level, msg)
     if not isclass(exception) or not issubclass(exception, Exception):
         exception = Exception
-    raise exception(msg)
+    raise exception(msg)  # pylint: disable=W0719
 
 
 def bool2str(value):

requirements-dev.txt

@@ -10,9 +10,10 @@ docformatter
 flake8
 isort>5.5,!=5.11.0
 mock>4
-# ignore 2.12 bad docstring asterisks args handling (https://github.com/PyCQA/pylint/issues/5406)
-pylint>=2.11,!=2.12,!=2.15
-pylint-quotes
+# FIXME: use temporary unofficial version working with pylint>3 (https://github.com/edaniszewski/pylint-quotes/pull/30)
+pylint>=3
+pylint_quotes @ git+https://github.com/marekhanus/pylint-quotes.git@0.3.0a2
+pylint-per-file-ignores
 # bird-house/twticher, must match version in Dockerfile.adapater
 pyramid-twitcher>=0.11.0
 pytest

requirements-doc.txt

@@ -3,9 +3,9 @@
 # we actually need to install all requirements during docs build because of OpenAPI generation
 # (see 'docs/conf.py')
 -r requirements.txt
-astroid>=4.1
 pycodestyle>=2.6.0,<3
-sphinx-autoapi>=3.7.0
+sphinx-autoapi>=3.6.1,<3.7.0; python_version < "3.12"
+sphinx-autoapi>=3.6.1; python_version >= "3.12"
 sphinx-paramlinks>=0.4.1
 sphinx>=7,<8
 sphinxcontrib-redoc>=1.6.0

tests/interfaces.py

@@ -21,6 +21,7 @@
 from webtest.app import TestApp
 
 from magpie import __meta__
+from magpie.adapter import MagpieAdapter
 from magpie.api import schemas as s
 from magpie.api.webhooks import webhook_update_error_status
 from magpie.constants import MAGPIE_ROOT, get_constant
@@ -53,10 +54,6 @@
 from tests import runner, utils
 from tests.utils import TestVersion, check_network_mode
 
-if six.PY3:
-    # WARNING: Twitcher does not support Python 2 since 0.4.0, adapter cannot work without it
-    from magpie.adapter import MagpieAdapter
-
 if TYPE_CHECKING:
     # pylint: disable=W0611,unused-import
     from typing import Dict, List, Optional, Set, Tuple, Union

tests/test_adapter.py

@@ -16,6 +16,7 @@
 from six.moves.urllib.parse import urlparse
 
 from magpie import __meta__
+from magpie.adapter.magpieowssecurity import MagpieOWSSecurity, OWSAccessForbidden
 from magpie.constants import get_constant
 from magpie.models import Route
 from magpie.permissions import Access, Permission, PermissionSet, Scope
@@ -26,9 +27,6 @@
 
 from twitcher.__version__ import __version__ as twitcher_version  # noqa
 
-if six.PY3:
-    from magpie.adapter.magpieowssecurity import MagpieOWSSecurity, OWSAccessForbidden  # noqa: F401
-
 if TYPE_CHECKING:
     from typing import Any, Callable, Tuple
 

tests/test_services.py

@@ -19,6 +19,7 @@
 from sqlalchemy import inspect as sa_inspect
 
 from magpie import __meta__, models, owsrequest
+from magpie.adapter.magpieowssecurity import OWSAccessForbidden  # noqa  # defined via Twitcher
 from magpie.constants import get_constant
 from magpie.permissions import Access, Permission, PermissionSet, PermissionType, Scope
 from magpie.services import (
@@ -34,9 +35,6 @@
 from tests import interfaces as ti
 from tests import runner, utils
 
-if six.PY3:
-    from magpie.adapter.magpieowssecurity import OWSAccessForbidden  # noqa  # defined via Twitcher
-
 if TYPE_CHECKING:
     # pylint: disable=W0611,unused-import
     from typing import Dict, Optional, Tuple, Union

tests/utils.py

@@ -566,7 +566,7 @@ def webhook_app():
         except OSError as exception:
             if exception.errno == EADDRINUSE:
                 # The app is already running, we just need to reset the webhook status and saved payload for a new test.
-                resp = requests.post(webhook_url + "/reset")
+                resp = requests.post(webhook_url + "/reset", timeout=10)
                 check_response_basic_info(resp, 200, expected_type=CONTENT_TYPE_HTML, expected_method="POST")
                 return
             raise
@@ -1249,7 +1249,7 @@ def get_session_user(app_or_url, headers=None):
     if isinstance(app_or_url, TestApp):
         resp = app_or_url.get("/session", headers=headers)
     else:
-        resp = requests.get("{}/session".format(app_or_url), headers=headers)
+        resp = requests.get("{}/session".format(app_or_url), headers=headers, timeout=10)
     if resp.status_code != 200:
         raise Exception("cannot retrieve logged-in user information")
     return resp
@@ -1305,7 +1305,7 @@ def check_or_try_login_user(test_item,                      # type: AnyMagpieTes
                 resp = app_or_url.post_json("/signin", data, headers=headers)
                 resp_cookies = app_or_url.cookies  # automatically set by TestApp processing
         else:
-            resp = requests.post("{}/signin".format(app_or_url), json=data, headers=headers)
+            resp = requests.post("{}/signin".format(app_or_url), json=data, headers=headers, timeout=10)
             resp_cookies = resp.cookies
 
         # response OK (200) if directly from API /signin