Modernize python packaging

.github/PULL_REQUEST_TEMPLATE.md

@@ -25,7 +25,7 @@ Fixes #{issue_number}
 > Make sure all existing unit tests pass.
 > You can run them locally using:
 > 
->     pytest smart_open
+>     pytest tests
 > 
 > If there are any failures, please fix them before creating the PR (or mark it as WIP, see below).
 

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+- package-ecosystem: github-actions
+  directory: /
+  schedule:
+    interval: monthly
+  groups:
+    github-actions:
+      patterns:
+      - '*'

.github/workflows/python-package.yml

@@ -1,16 +1,22 @@
-name: Test
-on: [push, pull_request]
+name: CI
+on:
+  pull_request:
+  push:
+    branches: [master, develop]
+  workflow_dispatch:  # allows running CI manually from the Actions tab
 concurrency:  # https://stackoverflow.com/questions/66335225#comment133398800_72408109
   group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 jobs:
   linters:
     runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # fetch git tags for setuptools_scm (smart_open.__version__)
 
       - name: Setup up Python 3.11
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: "3.11"
 
@@ -46,9 +52,11 @@ jobs:
           - {python-version: '3.12', os: windows-2025}
           - {python-version: '3.13', os: windows-2025}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # fetch git tags for setuptools_scm (smart_open.__version__)
 
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -62,7 +70,7 @@ jobs:
         run: pip install -e .[test]
 
       - name: Run unit tests
-        run: pytest smart_open -v -rfxECs --durations=20
+        run: pytest tests -v -rfxECs --durations=20
 
   doctest:
     needs: [linters,unit_tests]
@@ -89,9 +97,11 @@ jobs:
           # - {python-version: '3.13', os: windows-2025}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # fetch git tags for setuptools_scm (smart_open.__version__)
 
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -127,9 +137,11 @@ jobs:
           # - {python-version: '3.13', os: windows-2025}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # fetch git tags for setuptools_scm (smart_open.__version__)
 
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -177,9 +189,11 @@ jobs:
           # - {python-version: '3.13', os: windows-2025}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # fetch git tags for setuptools_scm (smart_open.__version__)
 
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 

.github/workflows/release.yml

@@ -1,45 +1,47 @@
 name: Release to PyPI
 
 on:
-  push:
-    tags:
-      - 'v*.*.*'
+  release:
+    types: [prereleased, released]
+
 jobs:
-  tarball:
-    if: github.event_name == 'push'
-    timeout-minutes: 1
-    runs-on: ubuntu-20.04
-    env:
-      PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
-      PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write  # softprops/action-gh-release
+      id-token: write  # pypa/gh-action-pypi-publish
+      issues: write # apexskier/github-release-commenter
+      pull-requests: write # apexskier/github-release-commenter
+
     steps:
-      - uses: actions/checkout@v1
-
-      - uses: actions/setup-python@v1
-        with:
-          python-version: "3.8.x"
-
-      # https://github.community/t/how-to-get-just-the-tag-name/16241/4
-      - name: Extract the version number
-        id: get_version
-        run: |
-          echo ::set-output name=V::$(python smart_open/version.py)
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          python -m venv venv
-          . venv/bin/activate
-          pip install twine wheel
-
-      - name: Build and upload tarball to PyPI
-        run: |
-          . venv/bin/activate
-          python setup.py sdist
-          twine upload dist/smart_open-${{ steps.get_version.outputs.V }}.tar.gz -u ${{ env.PYPI_USERNAME }} -p ${{ env.PYPI_PASSWORD }}
-
-      - name: Build and upload wheel to PyPI
-        run: |
-          . venv/bin/activate
-          python setup.py bdist_wheel
-          twine upload dist/smart_open-${{ steps.get_version.outputs.V }}-py3-none-any.whl -u ${{ env.PYPI_USERNAME }} -p ${{ env.PYPI_PASSWORD }}
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0 # fetch git tags for setuptools_scm (smart_open.__version__)
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.x
+
+    - name: Install dependencies
+      run: |
+        pip install --upgrade pip setuptools wheel build
+
+    - name: Build package distributions
+      run: |
+        python -m build
+
+    - name: Upload package distributions as release assets
+      uses: softprops/[email protected]
+      with:
+        files: dist/*
+
+    # https://github.com/pypa/gh-action-pypi-publish#trusted-publishing
+    - name: Publish package distributions to PyPI
+      uses: pypa/[email protected]
+
+    - uses: apexskier/[email protected]
+      with:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        comment-template: |
+          Released {release_link}

MANIFEST.in

@@ -1,4 +1,4 @@
-include LICENSE
-include README.rst
-include MIGRATING_FROM_OLDER_VERSIONS.rst
-include CHANGELOG.md
+global-exclude *
+include pyproject.toml
+graft smart_open
+global-exclude *.pyc

README.rst

@@ -508,7 +508,7 @@ Before you can run the test suite, install the test dependencies::
 
 Now, you can run the unit tests::
 
-    pytest smart_open
+    pytest tests
 
 The tests are also run automatically with `GitHub Actions <https://github.com/piskvorky/smart_open/actions/workflows/python-package.yml>`_ on every commit push & pull request.
 

ci_helpers/test_missing_dependencies.py

@@ -4,7 +4,7 @@
 os.environ['SMART_OPEN_TEST_MISSING_DEPS'] = '1'
 command = [
     'pytest',
-    'smart_open/tests/test_package.py',
+    'tests/test_package.py',
     '-v',
     '--cov', 'smart_open',
     '--cov-report', 'term-missing',

help.txt

@@ -32,10 +32,8 @@ PACKAGE CONTENTS
     s3
     smart_open_lib
     ssh
-    tests (package)
     transport
     utils
-    version
     webhdfs
 
 FUNCTIONS

pyproject.toml

@@ -1,2 +1,58 @@
-[tool.pytest.ini_options]
-testpaths = ["smart_open"]
+[build-system]
+requires = ["setuptools>=64", "setuptools_scm>=8"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]
+fallback_version = "0.1.dev1+gitnotfound"
+
+[project]
+name = "smart_open"
+description = "Utils for streaming large files (S3, HDFS, GCS, SFTP, Azure Blob Storage, gzip, bz2, zst...)"
+readme = "README.rst"
+urls = {Repository = "https://github.com/piskvorky/smart_open"}
+authors = [{name = "Radim Rehurek", email = "[email protected]"}]
+requires-python = ">=3.7,<4.0" # sync with classifiers below
+classifiers = [
+  "Development Status :: 5 - Production/Stable",
+  "Environment :: Console",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: MIT License",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.7",
+  "Programming Language :: Python :: 3.8",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Topic :: System :: Distributed Computing",
+  "Topic :: Database :: Front-Ends",
+]
+dynamic = ["version"]
+dependencies = ["wrapt"]
+
+[project.optional-dependencies]
+s3 = ["boto3"]
+gcs = ["google-cloud-storage>=2.6.0"]
+azure = ["azure-storage-blob", "azure-common", "azure-core"]
+http = ["requests"]
+webhdfs = ["requests"]
+ssh = ["paramiko"]
+zst = ["zstandard"]
+all = ["smart_open[s3,gcs,azure,http,webhdfs,ssh,zst]"]
+test = [
+    "smart_open[all]",
+    "moto[server]",
+    "responses",
+    "pytest",
+    "pytest-rerunfailures",
+    "pytest_benchmark",
+    "awscli",
+    "pyopenssl",
+    "numpy",
+]
+
+[tool.setuptools.packages.find]
+include = ["smart_open"]

release/README.md

@@ -1,6 +1,6 @@
 # Release Scripts
 
-This subdirectory contains various scripts for making a smart_open release.
+This subdirectory contains various scripts for maintainers.
 
 ## Prerequisites
 
@@ -17,30 +17,19 @@ All of the above are generally freely available, e.g. installable via apt in Ubu
 
 ## Release Procedure
 
-First, check that the [latest commit](https://github.com/RaRe-Technologies/smart_open/commits/master) passed all CI.
-
-For the subsequent steps to work, you will need to be in the top-level subdirectory for the repo (e.g. /home/misha/git/smart_open).
-
-Prepare the release, replacing 2.3.4 with the actual version of the new release:
-
-    bash release/prepare.sh 2.3.4
-
-This will create a local release branch.
-Look around the branch and make sure everything is in order.
-Checklist:
-
-- [ ] Does smart_open/version.py contain the correct version number for the release?
-- [ ] Does the CHANGELOG.md contain a section detailing the new release?
-- [ ] Are there any PRs that should be in CHANGELOG.md, but currently aren't?
-
-If anything is out of order, make the appropriate changes and commit them to the release branch before proceeding.
-
-**This is the point of no return**.
-**Once you're happy with the release branch**, run:
-
-    bash release/merge.sh
-
-Congratulations, at this stage, you are done!
+> ![New GitHub Release](https://github.com/user-attachments/assets/cf8f2fa4-37c1-4e50-9fd8-ab6e3fd705b5)
+> *New GitHub Release dialog*
+
+- Check that the [latest commit](https://github.com/piskvorky/smart_open/commits/develop) on `develop` passed all CI.
+- Run `bash release/release.sh` to update `CHANGELOG.md` and then update `master` branch.
+- Create a [new GitHub Release](https://github.com/piskvorky/smart_open/releases/new).
+    - Fill in the new version including a `v` prefix and press enter.
+    - Confirm that it reads "Excellent! This tag will be created from the target when you publish this release.".
+    - Select target branch `master`.
+    - Click "Generate release notes" on the right top.
+    - Click "Publish release".
+    - The GitHub Release and corresponding git tag gets created on the merge commit on `master`.
+    - GitHub Actions [`release.yml`](https://github.com/piskvorky/smart_open/actions/workflows/release.yml) is triggered, and uploads distributions to [PyPI](https://pypi.org/project/smart-open/) and to the new [GitHub Release](https://github.com/piskvorky/smart_open/releases).
 
 ## Troubleshooting