Merge branch 'master' into stable

Change-Id: I05fdcfb8ded736af5ef0bc434c4f410005cbac7b

.github/workflows/doctest.yml

@@ -60,6 +60,7 @@ jobs:
         python -m pip install --upgrade pip
         pip --version
         pip install coverage
+        pip install "tomli; python_version < '3.11'"
         pip install "importlib_metadata ; python_version < '3.8'"
         pip install mwparserfromhell
         pip install packaging

.github/workflows/login_tests-ci.yml

@@ -86,6 +86,7 @@ jobs:
         python -m pip install --upgrade pip
         pip --version
         pip install coverage
+        pip install "tomli; python_version < '3.11'"
         pip install "importlib_metadata ; python_version < '3.8'"
         pip install mwparserfromhell
         pip install packaging

.github/workflows/oauth_tests-ci.yml

@@ -84,8 +84,12 @@ jobs:
         python -m pip install --upgrade pip
         pip --version
         pip install coverage
+        # tomli required for coverage due to T380697
+        pip install "tomli; python_version < '3.11'"
         pip install "importlib_metadata ; python_version < '3.8'"
         pip install mwparserfromhell
+        # PyJWT added due to T380270
+        pip install "PyJWT != 2.10.0, != 2.10.1 ; python_version > '3.8'"
         pip install mwoauth
         pip install packaging
         pip install requests

.github/workflows/pywikibot-ci.yml

@@ -89,6 +89,8 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip --version
+        # T380732
+        pip install "coverage !=7.6.2,!=7.6.3,!=7.6.4,!=7.6.5,!=7.6.6,!=7.6.7,!=7.6.8; implementation_name=='pypy' and python_version=='3.10'"
         if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
         pip install wikitextparser

.pre-commit-config.yaml

@@ -29,16 +29,15 @@ repos:
       - id: destroyed-symlinks
       - id: end-of-file-fixer
       - id: fix-byte-order-marker
-      - id: fix-encoding-pragma
-        args:
-          - --remove
+        exclude: '^tests/data/'
       - id: forbid-new-submodules
       - id: mixed-line-ending
       - id: pretty-format-json
         args:
           - --autofix
           - --indent=4
           - --no-ensure-ascii
+        exclude: pywikibot/scripts/i18n/pywikibot/
       - id: trailing-whitespace
         args:
           - --markdown-linebreak-ext=rst
@@ -50,6 +49,12 @@ repos:
       - id: python-check-blanket-type-ignore
       - id: python-check-mock-methods
       - id: python-use-type-annotations
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.8.0
+    hooks:
+      - id: ruff
+        args:
+        - --fix
   - repo: https://github.com/asottile/pyupgrade
     rev: v3.19.0
     hooks:
@@ -72,6 +77,11 @@ repos:
     hooks:
       - id: isort
         exclude: '^pwb\.py$'
+  - repo: https://github.com/jshwi/docsig
+    rev: v0.65.0
+    hooks:
+      - id: docsig
+        exclude: ^(tests|scripts)
   - repo: https://github.com/PyCQA/flake8
     rev: 7.1.1
     hooks:
@@ -80,15 +90,11 @@ repos:
           - --doctests
         additional_dependencies:
           # Due to incompatibilities between packages the order matters.
-          - darglint2
-          - pydocstyle==6.3.0  # deprecated and no longer maintained
           - flake8-bugbear!=24.1.17
           - flake8-comprehensions>=3.13.0
-          - flake8-docstrings>=1.4.0
           - flake8-mock-x2
           - flake8-print>=5.0.0
           - flake8-quotes>=3.3.2
           - flake8-raise
           - flake8-tuple>=0.4.1
-          - flake8-no-u-prefixed-strings>=0.2
           - pep8-naming>=0.13.3

CONTENT.rst

@@ -10,8 +10,6 @@ The contents of the package
     +---------------------------+-----------------------------------------------------------+
     | CONTENT.rst               | This Content description file                             |
     +---------------------------+-----------------------------------------------------------+
-    | dev-requirements.txt      | PIP requirements file for development dependencies        |
-    +---------------------------+-----------------------------------------------------------+
     | Dockerfile                | Assemble a Docker image, install all dependencies via pip |
     +---------------------------+-----------------------------------------------------------+
     | Dockerfile-dev            | Docker image including development dependencies           |
@@ -20,18 +18,22 @@ The contents of the package
     +---------------------------+-----------------------------------------------------------+
     | LICENSE                   | Reference to the MIT license                              |
     +---------------------------+-----------------------------------------------------------+
-    | make_dist.py              | Script to create a Pywikibot distribution                 |
-    +---------------------------+-----------------------------------------------------------+
     | MANIFEST.in               | Setup file for package data                               |
     +---------------------------+-----------------------------------------------------------+
+    | README.rst                | Short info string used by Pywikibot Nightlies             |
+    +---------------------------+-----------------------------------------------------------+
+    | ROADMAP.rst               | PyPI version roadmap file                                 |
+    +---------------------------+-----------------------------------------------------------+
+    | dev-requirements.txt      | PIP requirements file for development dependencies        |
+    +---------------------------+-----------------------------------------------------------+
+    | make_dist.py              | Script to create a Pywikibot distribution                 |
+    +---------------------------+-----------------------------------------------------------+
     | pwb.py                    | Caller script for pwb wrapper script                      |
     +---------------------------+-----------------------------------------------------------+
-    | README.rst                | Short info string used by Pywikibot Nightlies             |
+    | pyproject.toml            | Configuration file used by packaging tools and tests      |
     +---------------------------+-----------------------------------------------------------+
     | requirements.txt          | General PIP requirements file                             |
     +---------------------------+-----------------------------------------------------------+
-    | ROADMAP.rst               | PyPI version roadmap file                                 |
-    +---------------------------+-----------------------------------------------------------+
     | setup.py                  | Installer script for Pywikibot framework                  |
     +---------------------------+-----------------------------------------------------------+
     | tox.ini                   | Tests config file                                         |
@@ -44,6 +46,8 @@ The contents of the package
     +---------------------------+-----------------------------------------------------------+
     | Directories                                                                           |
     +===========================+===========================================================+
+    | docs                      | Documentation files                                       |
+    +---------------------------+-----------------------------------------------------------+
     | pywikibot                 | Contains some libraries and control files                 |
     +---------------------------+-----------------------------------------------------------+
     | scripts                   | Contains all bots and utility scripts                     |

HISTORY.rst

@@ -1,6 +1,21 @@
 Release History
 ===============
 
+9.5.0
+-----
+*30 October 2024*
+
+* Add support for tcywikisource and tcywiktionary (:phab:`T378473`, :phab:`T378465`)
+* i18n-updates
+* Update invisible chars in :mod:`tools.chars` from unicode 16.0.0
+* Rename :meth:`DataSite.getPropertyType()<pywikibot.site._datasite.DataSite.getPropertyType>`
+  to :meth:`DataSite.get_property_type()<pywikibot.site._datasite.DataSite.get_property_type>`
+* provide XXXI with :func:`date.romanNumToInt` and :func:`date.intToRomanNum` functions
+* No longer raise :exc:`exceptions.UnsupportedPageError` within :meth:`data.api.PageGenerator.result` (:phab:`T377651`)
+* Extract messages with strong tag from xtools as error message in
+  :meth:`Page.authorship()<page._toolforge.WikiBlameMixin.authorship>` (:phab:`T376815`)
+
+
 9.4.1
 -----
 *15 October 2024*
@@ -652,7 +667,7 @@ Release History
 * Raise InvalidTitleError instead of unspecific ValueError in ProofreadPage (:phab:`T308016`)
 * Preload pages if GeneratorFactory.articlenotfilter_list is not empty; also set attribute ``is_preloading``.
 * ClaimCollection.toJSON() should not ignore new claim (:phab:`T308245`)
-* use linktrail via siteinfo and remove `update_linkrtrails` maintenance script
+* use linktrail via siteinfo and remove `update_linktrails` maintenance script
 * Print counter statistic for all counters (:phab:`T307834`)
 * Use proofreadpagesinindex query module
 * Prioritize -namespaces options in `pagegenerators.handle_args` (:phab:`T222519`)

ROADMAP.rst

@@ -1,19 +1,31 @@
 Current Release Changes
 =======================
 
-* Add support for tcywikisource and tcywiktionary (:phab:`T378473`, :phab:`T378465`)
+* Add support for idwikivoyage (:phab:`T381082`)
+* Add docstrings of :class:`tools.classproperty` methods (:phab:`T380628`)
+* Site property :attr:`BaseSite.codes<pywikibot.site._basesite.BaseSite.codes>` was added (:phab:`T380606`)
+* Increase *leeway* parameter of :meth:`login.OauthLoginManager.identity` (:phab:`T380270`)
+* Show a warning if *ignore_extension* parameter of :class:`pywikibot.FilePage` was set and the extension is invalid
+* Remove old code of Python 3.2 or older in :func:`tools.chars.replace_invisible` due to :pep:`393`
+* use :meth:`BasePage.autoFormat()<page.BasePage.autoFormat>` instead of :func:`date.getAutoFormat` in
+  :mod:`titletranslate`
+* Upcast :class:`pywikibot.Page` to :class:`pywikibot.FilePage` in :meth:`PageGenerator.result()
+  <data.api.PageGenerator.result>` if ``imageinfo`` is given (:phab:`T379513`)
+* Update oauth requirements
 * i18n-updates
-* Update invisible chars in :mod:`tools.chars` from unicode 16.0.0
-* Rename :meth:`DataSite.getPropertyType()<pywikibot.site._datasite.DataSite.getPropertyType>`
-  to :meth:`DataSite.get_property_type()<pywikibot.site._datasite.DataSite.get_property_type>`
-* provide XXXI with :func:`date.romanNumToInt` and :func:`date.intToRomanNum` functions
-* No longer raise :exc:`exceptions.UnsupportedPageError` within :meth:`data.api.PageGenerator.result` (:phab:`T377651`)
-* Extract messages with strong tag from xtools as error message in
-  :meth:`Page.authorship()<page._toolforge.WikiBlameMixin.authorship>` (:phab:`T376815`)
+* Implement param *with_sort_key* in :meth:`page.BasePage.categories` (:phab:`T75561`)
+* Python 3.7 support will be discontinued and probably this is the last version supporting it
+* Add :meth:`page.BasePage.get_revision` method
+* Retry :meth:`data.sparql.SparqlQuery.query` on internal server error (500) (:phab:`T378788`)
+* Extract :meth:`APISite.linktrail()<pywikibot.site._apisite.APISite.linktrail>`
+  for hr-wiki (:phab:`T378787`)
+
 
 Current Deprecations
 ====================
 
+* 9.6.0: :meth:`BaseSite.languages()<pywikibot.site._basesite.BaseSite.languages>` will be removed in favour of
+  :attr:`BaseSite.codes<pywikibot.site._basesite.BaseSite.codes>`
 * 9.5.0: :meth:`DataSite.getPropertyType()<pywikibot.site._datasite.DataSite.getPropertyType>` will be removed
   in favour of :meth:`DataSite.get_property_type()<pywikibot.site._datasite.DataSite.get_property_type>`
 * 9.4.0: :mod:`flow` support is deprecated and will be removed (:phab:`T371180`)
@@ -71,6 +83,7 @@ Current Deprecations
 Pending removal in Pywikibot 10
 -------------------------------
 
+* 9.6.0: Python 3.7 support is deprecated and will be dropped with Pywikibot 10
 * 9.1.0: :func:`version.svn_rev_info` and :func:`version.getversion_svn` will be removed. SVN is no longer supported.
   (:phab:`T362484`)
 * 7.7.0: :mod:`tools.threading` classes should no longer imported from :mod:`tools`
@@ -81,7 +94,7 @@ Pending removal in Pywikibot 10
 * 7.3.0: Old color escape sequences like ``\03{color}`` is deprecated in favour of new color format like <<color>>
 * 7.3.0: ``linktrail`` method of :class:`family.Family` is deprecated; use :meth:`APISite.linktrail()
   <pywikibot.site._apisite.APISite.linktrail>` instead
-* 7.2.0: Positional arguments *decoder*, *layer* and *newline* for :mod:`logging` functions where dropped; keyword
+* 7.2.0: Positional arguments *decoder*, *layer* and *newline* for :mod:`logging` functions were dropped; keyword
   arguments must be used instead.
 * 7.2.0: ``tb`` parameter of :func:`exception()<pywikibot.logging.exception>` function was renamed to ``exc_info``
 * 7.2.0: XMLDumpOldPageGenerator is deprecated in favour of a ``content`` parameter of

dev-requirements.txt

@@ -10,6 +10,9 @@ pytest-xvfb>=3.0.0
 
 pre-commit
 coverage>=5.2.1
+# required for coverage (T380697)
+tomli>=2.0.1; python_version < "3.11"
+
 
 # optional but needed for tests
 fake-useragent

docs/api_ref/family.rst

@@ -4,6 +4,7 @@
 
 .. automodule:: family
    :synopsis: Objects representing MediaWiki families
+   :exclude-members: Family
 
    .. autoclass:: Family
 

docs/api_ref/pywikibot.site.rst

@@ -27,7 +27,7 @@
          .. seealso:: :meth:`family.Family.linktrail`
          .. deprecated:: 7.3
             Only supported as :class:`APISite<pywikibot.site._apisite.APISite>`
-            method. Use :meth:`APISite.linktrail
+            method. Use :meth:`APISite.linktrail()
             <pywikibot.site._apisite.APISite.linktrail>`
 
          :rtype: str

docs/conf.py

@@ -579,35 +579,10 @@ def pywikibot_script_docstring_fixups(app, what, name, obj, options, lines):
             length = 0
 
 
-def pywikibot_family_classproperty_getattr(obj, name, *defargs):
-    """Custom getattr() to get classproperty instances."""
-    from sphinx.util.inspect import safe_getattr
-
-    from pywikibot.family import Family
-    from pywikibot.tools import classproperty
-
-    if not isinstance(obj, type) or not issubclass(obj, Family):
-        return safe_getattr(obj, name, *defargs)
-
-    for base_class in obj.__mro__:
-        try:
-            prop = base_class.__dict__[name]
-        except KeyError:
-            continue
-
-        if not isinstance(prop, classproperty):
-            return safe_getattr(obj, name, *defargs)
-
-        return prop
-
-    return safe_getattr(obj, name, *defargs)
-
-
 def setup(app):
     """Implicit Sphinx extension hook."""
     app.connect('autodoc-process-docstring', pywikibot_docstring_fixups)
     app.connect('autodoc-process-docstring', pywikibot_script_docstring_fixups)
-    app.add_autodoc_attrgetter(type, pywikibot_family_classproperty_getattr)
 
 
 autoclass_content = 'both'

docs/index.rst

@@ -21,7 +21,8 @@ system that has a compatible version of Python installed. To check
 whether you have Python installed and to find its version, just type
 ``python`` at the CMD or shell prompt.
 
-Python 3.7 or higher is currently required to run the bot.
+Python 3.7 or higher is currently required to run the bot but Python 3.8 or
+higher is recommended. Python 3.7 support will be dropped with Pywikibot 10 soon.
 
 Pywikibot and this documentation are licensed under the
 :ref:`MIT license`;

docs/requirements.txt

@@ -6,5 +6,5 @@ rstcheck >=6.2.4
 sphinxext-opengraph >= 0.9.1
 sphinx-copybutton >= 0.5.2
 sphinx-tabs >= 3.4.7
-tomli >= 2.02; python_version < '3.11'
+tomli >= 2.0.1; python_version < '3.11'
 furo >= 2024.8.6

docs/scripts/archive.rst

@@ -3,12 +3,13 @@ Outdated core scripts
 *********************
 
 This list contains outdated scripts from :term:`core` banch which
-aren't supported any longer. They are either archived or deleted.
+aren't supported any longer. They aredeleted from repository.
 
-Feel free to reactivate any script at any time by creating a Phabricator
-task: :phab:`Recovery request
-<maniphest/task/edit/form/1/?projects=pywikibot,pywikibot-scripts&title=Recover
-Pywikibot%20script:%20>`
+.. hint::
+   Feel free to reactivate any script at any time by creating a
+   Phabricator task: :phab:`Recovery request
+   <maniphest/task/edit/form/1/?projects=pywikibot,pywikibot-scripts&title=Recover
+   Pywikibot%20script:%20>`
 
 .. seealso:: :ref:`Outdated compat scripts`