Modernize project to support install with uv

Author: jbcoe

.github/workflows/lint_python.yml

@@ -11,11 +11,15 @@ jobs:
     name: Check Python linting
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-python@v5
+      - name: Install the latest version of uv.
+        uses: astral-sh/setup-uv@v5
         with:
-          python-version: '3.13'
+          python-version: "3.13"
+
       - uses: actions/checkout@v4
-      - run: pip install -q uv && uv pip install --system ruff==0.4.3 isort
-      - run: ruff format --check --diff .
-      - run: ruff check
-      - run: isort -c --diff nle/
+        with:
+          submodules: true
+      - run: uv sync
+      - run: uv run ruff format --check --diff .
+      - run: uv run ruff check
+      - run: uv run isort -c --diff nle/

.github/workflows/test_and_deploy.yml

@@ -16,13 +16,15 @@ jobs:
         os: [ubuntu-latest, macos-latest]
       fail-fast: false
     steps:
-      - name: Setup Python ${{ matrix.python-version }} env
-        uses: actions/setup-python@v5
+      - name: Install the latest version of uv.
+        uses: astral-sh/setup-uv@v5
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Ensure latest pip, wheel & setuptools
-        run: python -m pip install -q --upgrade pip wheel setuptools
-      - name: Install dependencies
+      - name: Install the latest version of uv.
+        uses: astral-sh/setup-uv@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install non-python build dependencies
         run: |
           if [ "$RUNNER_OS" == "Linux" ]; then
             sudo apt-get update && sudo apt-get install -yq \
@@ -36,10 +38,11 @@ jobs:
               software-properties-common
           fi
       - uses: actions/checkout@v4
-      - name: Install from repo in test mode
-        run: pip install -e '.[dev]'
-      - name: Run tests
-        run: python -m pytest --import-mode=append -svx nle/tests
+      - name: Install package using uv
+        run: uv sync
+      - name: Run tests using uv
+        run: uv run --no-sync pytest --import-mode=append -svx
+        
   test_sdist_3_10:
     name: Test sdist on MacOS w/ Py3.10
     needs: test_repo
@@ -51,6 +54,9 @@ jobs:
           python-version: '3.10'
       - name: Ensure latest pip, wheel & setuptools
         run: python -m pip install -q --upgrade pip wheel setuptools
+      - name: Install dependencies
+        run: |
+          brew install cmake
       - uses: actions/checkout@v4
       - name: Generate sdist
         run: |
@@ -77,6 +83,9 @@ jobs:
           python-version: 3.13
       - name: Ensure latest pip, wheel & setuptools
         run: python -m pip install -q --upgrade pip wheel setuptools
+      - name: Install dependencies
+        run: |
+          brew install cmake
       - uses: actions/checkout@v4
       - name: Generate sdist
         run: |

nle/agent/agent.py

@@ -104,7 +104,7 @@
 
 logging.basicConfig(
     format=(
-        "[%(levelname)s:%(process)d %(module)s:%(lineno)d %(asctime)s] " "%(message)s"
+        "[%(levelname)s:%(process)d %(module)s:%(lineno)d %(asctime)s] %(message)s"
     ),
     level=logging.INFO,
 )

nle/dataset/populate_db.py

@@ -151,7 +151,7 @@ def add_altorg_directory(path, name, filename=nld.db.DB):
             game_gen = game_data_generator(xlogfile, separator=sep)
             insert_sql = f"""
                 INSERT INTO games
-                VALUES (NULL, {','.join('?' for _ in XLOGFILE_COLUMNS)} )
+                VALUES (NULL, {",".join("?" for _ in XLOGFILE_COLUMNS)} )
             """
             c.executemany(insert_sql, game_gen)
 
@@ -293,7 +293,7 @@ def add_nledata_directory(path, name, filename=nld.db.DB):
             game_gen = game_data_generator(xlogfile, filter=_filter)
             insert_sql = f"""
                 INSERT INTO games
-                VALUES (NULL, {','.join('?' for _ in XLOGFILE_COLUMNS)} )
+                VALUES (NULL, {",".join("?" for _ in XLOGFILE_COLUMNS)} )
             """
             c.executemany(insert_sql, game_gen)
             gameids = nld.db.get_most_recent_games(c.rowcount, conn=c)

nle/scripts/ttyrec.py

@@ -15,7 +15,7 @@
     "-1",
     "--no_input",
     action="store_true",
-    help=("Save in ttyrec (not ttyrec2) format, " "without input data"),
+    help=("Save in ttyrec (not ttyrec2) format, without input data"),
 )
 parser.add_argument(
     "-e", "--execute", metavar="CMD", help="command to run (default: /bin/sh)"

nle/tests/test_converter.py

@@ -9,38 +9,38 @@
 from nle.dataset import Converter
 
 # From
-#   https://alt.org/nethack/trd/?file=https://s3.amazonaws.com/altorg/ttyrec/Anarchos/2020-10-03.17:27:10.ttyrec.bz2  # noqa: B950
+#   https://alt.org/nethack/trd/?file=https://s3.amazonaws.com/altorg/ttyrec/Anarchos/2020-10-03.17:27:10.ttyrec.bz2
 TTYREC_2020 = "2020-10-03.17_27_10.ttyrec.bz2"
 COLSROWS = "2020-10-03.17_27_10.rowscols.txt"
 TIMESTAMPS = "2020-10-03.17_27_10.timestamps.txt.bz2"
 FINALFRAME = "2020-10-03.17_27_10.finalframe.txt"
 FINALFRAMECOLORS = "2020-10-03.17_27_10.finalframe.colors.txt"
 
 # From
-#  https://alt.org/nethack/trd/?file=https://s3.amazonaws.com/altorg/ttyrec/Qvazzler/2009-02-05.10:33:51.ttyrec.bz2  # noqa: B950
+#  https://alt.org/nethack/trd/?file=https://s3.amazonaws.com/altorg/ttyrec/Qvazzler/2009-02-05.10:33:51.ttyrec.bz2
 # This ttyrec uses IBMGraphics (code page 437)
 TTYREC_IBMGRAPHICS = "2009-02-05.10_33_51.ttyrec.bz2"
 TTYREC_IBMGRAPHICS_FRAME_10 = "2009-02-05.10_33_51.frame.10.txt"
 
 # From
-#  https://alt.org/nethack/trd/?file=https://s3.amazonaws.com/altorg/ttyrec/moo22/2018-09-27.00:20:39.ttyrec.bz2  # noqa: B950
+#  https://alt.org/nethack/trd/?file=https://s3.amazonaws.com/altorg/ttyrec/moo22/2018-09-27.00:20:39.ttyrec.bz2
 TTYREC_2018 = "2018-09-27.00_20_39.ttyrec.bz2"
 
 # From
-#  https://alt.org/nethack/trd/?file=https://s3.amazonaws.com/altorg/ttyrec/mackeyth/2017-07-28.18:44:17.ttyrec.bz2  # noqa: B950
+#  https://alt.org/nethack/trd/?file=https://s3.amazonaws.com/altorg/ttyrec/mackeyth/2017-07-28.18:44:17.ttyrec.bz2
 # This ttyrec uses DECGraphics (https://en.wikipedia.org/wiki/DEC_Special_Graphics)
 TTYREC_DECGRAPHICS = "2017-07-28.18_44_17.ttyrec.bz2"
 TTYREC_DECGRAPHICS_FRAME_10 = "2017-07-28.18_44_17.frame.10.txt"
 
 # From
-#  https://alt.org/nethack/trd/?file=https://s3.amazonaws.com/altorg/ttyrec/waIrus/2019-11-18.08:52:15.ttyrec.bz2  # noqa: B950
-# This ttyrec uses unknown control sequences - it looks like maybe VT420 with rectangle control. `ESC [ %d ; %d ; %d ; %d z`  # noqa: B950
+#  https://alt.org/nethack/trd/?file=https://s3.amazonaws.com/altorg/ttyrec/waIrus/2019-11-18.08:52:15.ttyrec.bz2
+# This ttyrec uses unknown control sequences - it looks like maybe VT420 with rectangle control. `ESC [ %d ; %d ; %d ; %d z`
 TTYREC_UNKGRAPHICS = "2019-11-18.08_52_15.ttyrec.bz2"
 TTYREC_UNKGRAPHICS_FRAME_10 = "2019-11-18.08_52_15.frame.10.txt"
 
 # From
-#  https://alt.org/nethack/trd/?file=https://s3.amazonaws.com/altorg/ttyrec/fare/2012-02-16.21:09:51.ttyrec.bz2  # noqa: B950
-# This ttyrec uses SHIFT IN and SHIFT OUT ASCII control sequences toggle DEC graphics set.  # noqa: B950
+#  https://alt.org/nethack/trd/?file=https://s3.amazonaws.com/altorg/ttyrec/fare/2012-02-16.21:09:51.ttyrec.bz2
+# This ttyrec uses SHIFT IN and SHIFT OUT ASCII control sequences toggle DEC graphics set.
 TTYREC_SHIFTIN = "2012-02-16.21_09_51.ttyrec.bz2"
 TTYREC_SHIFTIN_FRAME_10 = "2012-02-16.21_09_51.frame.10.txt"
 

pyproject.toml

@@ -1,3 +1,69 @@
+[project]
+name = "nle"
+description = "The NetHack Learning Environment (NLE): a reinforcement learning environment based on NetHack"
+readme = "README.md"
+authors = [ { name = "The NLE Dev Team" } ]
+requires-python = ">=3.10"
+dynamic = ["classifiers", "version"]
+dependencies = [
+  "pybind11>=2.2",
+  "numpy>=1.16",
+  "gymnasium==1.2.0"
+]
+
+[project.license]
+file = "LICENSE"
+
+[project.urls]
+Homepage = "https://github.com/NetHack-LE/nle"
+
+[build-system]
+requires = ["setuptools", "pybind11>=2.2"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages]
+find = { include = [
+    "nle",
+    "nle.dataset",
+    "nle.env",
+    "nle.nethack",
+    "nle.agent",
+    "nle.scripts",
+    "nle.tests"
+]}
+
+[project.scripts]
+nle-play = "nle.scripts.play:main"
+nle-ttyrec = "nle.scripts.ttyrec:main"
+nle-ttyplay = "nle.scripts.ttyplay:main"
+nle-ttyplay2 = "nle.scripts.ttyplay2:main"
+nle-read-tty = "nle.scripts.read_tty:main"
+
+[dependency-groups]
+dev = [
+  "nle[dev]",
+]
+
+[project.optional-dependencies]
+agent = [
+  "torch>=1.3.1"
+]
+dev = [
+  "pre-commit>=2.0.1",
+  "isort>=5.13.2",
+  "cmake_format>=0.6.10",
+  "memory-profiler>=0.60.0",
+  "pytest>=6.2.5",
+  "pytest-benchmark>=3.4.1",
+  "sphinx>=2.4.4",
+  "sphinx-rtd-theme>=0.4.3",
+  "setuptools>=69.5.1",
+  "ruff>=0.4.3",
+]
+all = [
+  "nle[agent,dev]"
+]
+
 [tool.black]
 line-length = 88
 target-version = ['py38']
@@ -68,3 +134,6 @@ manylinux-aarch64-image = "manylinux2014"
 [tool.cibuildwheel.macos]
 # For macOS wheels, we only need to install cmake
 before-all = "rm -rf {project}/build {project}/*.so {project}/CMakeCache.txt && brew install cmake"
+
+[tool.pytest.ini_options]
+testpaths = ["nle/tests"]

setup.py

@@ -4,9 +4,6 @@
 #
 # List of environment variables used:
 #
-#  NLE_PACKAGE_NAME
-#    Prefix of the generated package (defaults to "nle").
-#
 #  NLE_BUILD_RELEASE
 #    If set, builds wheel (s)dist such as to prepare it for upload to PyPI.
 #
@@ -71,48 +68,7 @@ def build_extension(self, ext):
             sys.exit(1)
 
 
-packages = [
-    "nle",
-    "nle.dataset",
-    "nle.env",
-    "nle.nethack",
-    "nle.agent",
-    "nle.scripts",
-    "nle.tests",
-]
-
-entry_points = {
-    "console_scripts": [
-        "nle-play = nle.scripts.play:main",
-        "nle-ttyrec = nle.scripts.ttyrec:main",
-        "nle-ttyplay = nle.scripts.ttyplay:main",
-        "nle-ttyplay2 = nle.scripts.ttyplay2:main",
-        "nle-read-tty = nle.scripts.read_tty:main",
-    ]
-}
-
-
-extras_deps = {
-    "dev": [
-        "pre-commit>=2.0.1",
-        "isort>=5.13.2",
-        "cmake_format>=0.6.10",
-        "memory-profiler>=0.60.0",
-        "pytest>=6.2.5",
-        "pytest-benchmark>=3.4.1",
-        "sphinx>=2.4.4",
-        "sphinx-rtd-theme>=0.4.3",
-        "setuptools>=69.5.1",
-        "ruff>=0.4.3",
-    ],
-    "agent": ["torch>=1.3.1"],
-}
-
-extras_deps["all"] = [item for group in extras_deps.values() for item in group]
-
-
 if __name__ == "__main__":
-    package_name = os.getenv("NLE_PACKAGE_NAME", "nle")
     cwd = os.path.dirname(os.path.abspath(__file__))
     version = open("version.txt", "r").read().strip()
     sha = "Unknown"
@@ -128,36 +84,18 @@ def build_extension(self, ext):
 
     if sha != "Unknown" and not os.getenv("NLE_RELEASE_BUILD"):
         version += "+" + sha[:7]
+    package_name = setuptools.find_packages()[0]
     print("Building wheel {}-{}".format(package_name, version))
 
     version_path = os.path.join(cwd, "nle", "version.py")
     with open(version_path, "w") as f:
         f.write("__version__ = '{}'\n".format(version))
         f.write("git_version = {}\n".format(repr(sha)))
 
-    with open("README.md") as f:
-        long_description = f.read()
-
     setuptools.setup(
-        name=package_name,
         version=version,
-        description=(
-            "The NetHack Learning Environment (NLE): "
-            "a reinforcement learning environment based on NetHack"
-        ),
-        long_description=long_description,
-        long_description_content_type="text/markdown",
-        author="The NLE Dev Team",
-        url="https://github.com/NetHack-LE/nle",
-        license="NetHack General Public License",
-        entry_points=entry_points,
-        packages=packages,
         ext_modules=[setuptools.Extension("nle", sources=[])],
         cmdclass={"build_ext": CMakeBuild},
-        setup_requires=["pybind11>=2.2"],
-        install_requires=["pybind11>=2.2", "numpy>=1.16", "gymnasium==1.2.0"],
-        extras_require=extras_deps,
-        python_requires=">=3.10",
         classifiers=[
             "License :: OSI Approved :: Nethack General Public License",
             "Development Status :: 5 - Production/Stable",