RF: switch from unmaintained fusepy to mfusepy

.github/workflows/docbuild.yml

@@ -18,7 +18,7 @@ jobs:
         python-version: '3.12'
     - name: Install dependencies
       run: |
-        # fusepy, imported to document the FUSE file system class, needs libfuse2
+        # mfusepy, imported to document the FUSE file system class, needs libfuse
         sudo apt-get update -qq
         sudo apt-get install -y libfuse2t64
         python -m pip install --upgrade pip

.github/workflows/test.yml

@@ -19,6 +19,7 @@ jobs:
       #   nonetwork — --no-network --no-forgejo (the CI default)
       #   full      — --libfuse (FUSE tests; network+forgejo enabled)
       #   libfuse   — --libfuse only
+      # fuse3: true runs against libfuse 3 instead of libfuse 2.
       matrix:
         include:
           - python-version: '3.10'
@@ -29,6 +30,7 @@ jobs:
             full: true
           - python-version: '3.14'
             libfuse: true
+            fuse3: true
     runs-on: ubuntu-22.04
     steps:
     - name: Set up system
@@ -42,7 +44,7 @@ jobs:
         sudo eatmydata apt-get install git-annex-standalone
     - name: Install libfuse (full and libfuse jobs)
       if: matrix.full || matrix.libfuse
-      run: sudo eatmydata apt-get install -y fuse
+      run: sudo eatmydata apt-get install -y ${{ matrix.fuse3 && 'fuse3' || 'fuse' }}
     - uses: actions/checkout@v7
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v7
@@ -54,6 +56,8 @@ jobs:
         python -m pip install --upgrade pip
         python -m pip install coverage datalad tox tox-gh-actions
     - name: Run tests
+      env:
+        FUSE_LIBRARY_NAME: ${{ matrix.fuse3 && 'fuse3' || '' }}
       run: |
         # give detailed info on actual test setup
         datalad wtf
@@ -70,4 +74,4 @@ jobs:
       with:
         fail_ci_if_error: false
         token: ${{ secrets.CODECOV_TOKEN }}
-        name: py${{ matrix.python-version }}${{ matrix.full && '-full' || '' }}${{ matrix.libfuse && '-libfuse' || '' }}
+        name: py${{ matrix.python-version }}${{ matrix.full && '-full' || '' }}${{ matrix.libfuse && '-libfuse' || '' }}${{ matrix.fuse3 && '-fuse3' || '' }}

.readthedocs.yaml

@@ -7,7 +7,7 @@ build:
   tools:
     python: "3.12"
   apt_packages:
-    # fusepy, imported to document the FUSE file system class, needs libfuse2
+    # mfusepy, imported to document the FUSE file system class, needs libfuse
     - libfuse2t64
   jobs:
     post_checkout:

CONTRIBUTING.md

@@ -27,7 +27,9 @@ tox -e py3
 
 ### FUSE mount tests
 
-Requires FUSE system libraries (`apt-get install fuse` on Debian/Ubuntu):
+Requires FUSE system libraries (`apt-get install fuse3` on Debian/Ubuntu).
+If libfuse 2 is installed as well, it is used unless `FUSE_LIBRARY_NAME=fuse3`
+is set:
 
 ```bash
 tox -e py3 -- --libfuse

README.md

@@ -19,8 +19,8 @@ or directly from Python.
     python3 -m pip install datalad-fuse
 
 [git-annex](https://git-annex.branchable.com/install/) is required, and FUSE
-for mounting datasets (e.g. `sudo apt-get install fuse3 libfuse2t64` on
-Ubuntu 24.04).  See the
+(libfuse 2 or 3) for mounting datasets (e.g. `sudo apt-get install fuse3` on
+Debian/Ubuntu).  See the
 [installation instructions](https://datalad-fuse.readthedocs.io/en/latest/installation.html)
 for details.
 

datalad_fuse/__init__.py

@@ -113,7 +113,7 @@ def __call__(
         allow_other: bool = False,
         caching: str | None = None,
     ) -> Iterator[Dict[str, Any]]:
-        from fuse import FUSE
+        from mfusepy import FUSE
 
         from .fuse_ import DataLadFUSE
 

datalad_fuse/fuse_.py

@@ -18,8 +18,8 @@
 
 from datalad import cfg
 from datalad.distribution.dataset import Dataset
-from fuse import FuseOSError, Operations
 import methodtools
+from mfusepy import FuseOSError, Operations
 
 from .consts import CACHE_SIZE
 from .fsspec import FsspecAdapter
@@ -47,6 +47,30 @@
 T = TypeVar("T")
 P = ParamSpec("P")
 
+# FUSE callbacks taking a path within the mount as their first argument
+_PATH_OPS = frozenset(
+    name
+    for name in dir(Operations)
+    if not name.startswith("_") and name != "init_with_config"
+)
+
+
+def _in_root(fs: DataLadFUSE, name: str, method: Callable[..., T]) -> Callable[..., T]:
+    """Map the FUSE path given to ``method`` into the dataset"""
+
+    @wraps(method)
+    def call(path: Optional[str], *args: Any) -> T:
+        lgr.debug("op=%s for path=%s with args %s", name, path, args)
+        if path is None:
+            # libfuse may pass no path for operations on an open handle
+            return method(path, *args)
+        if not fs.mode_transparent and ".git" in Path(path).parts:
+            lgr.debug("Raising ENOENT for .git")
+            raise FuseOSError(ENOENT)
+        return method(fs.root + path, *args)
+
+    return call
+
 
 def write_op(
     f: Callable[Concatenate[DataLadFUSE, str, P], T]
@@ -64,7 +88,7 @@ def wrapped(self: DataLadFUSE, path: str, *args: P.args, **kwargs: P.kwargs) ->
 
 
 class DataLadFUSE(Operations):  # LoggingMixIn,
-    """fusepy file system exposing a dataset, as used by ``datalad fusefs``
+    """mfusepy file system exposing a dataset, as used by ``datalad fusefs``
 
     Files are read via an :class:`~datalad_fuse.fsspec.FsspecAdapter`, so
     annexed files without local content are read from their remote URLs.
@@ -90,7 +114,7 @@ class DataLadFUSE(Operations):  # LoggingMixIn,
     --------
     Mount a dataset with extra FUSE options::
 
-        from fuse import FUSE
+        from mfusepy import FUSE
         FUSE(DataLadFUSE("/abs/path/to/ds", caching=False), "/mnt/point",
              foreground=True, ro=True)
     """
@@ -115,14 +139,19 @@ def __init__(
         # multiple times?
         self._counter = DataLadFUSE._counter_offset
 
-    def __call__(self, op: str, path: str, *args: Any) -> Any:
-        lgr.debug("op=%s for path=%s with args %s", op, path, args)
-        # if (".git", "annex", "objects") == Path(path).parts[-7:-4]:
-        #     import pdb; pdb.set_trace()
-        if not self.mode_transparent and ".git" in Path(path).parts:
-            lgr.debug("Raising ENOENT for .git")
-            raise FuseOSError(ENOENT)
-        return super(DataLadFUSE, self).__call__(op, self.root + path, *args)
+    def __getattribute__(self, name: str) -> Any:
+        # mfusepy invokes FUSE callbacks as plain method calls on this object
+        # (fusepy dispatched them via Operations.__call__), so this is the one
+        # place where paths coming from FUSE get mapped into the dataset.
+        # Hence callbacks must not be called on self from within this class.
+        value = super().__getattribute__(name)
+        if (
+            name in _PATH_OPS
+            and callable(value)
+            and not getattr(value, "libfuse_ignore", False)
+        ):
+            return _in_root(self, name, value)
+        return value
 
     def destroy(self, _path: Optional[str] = None) -> int:
         lgr.warning("Destroying fsspecs and collection of %d fhs", len(self._fhdict))
@@ -332,7 +361,7 @@ def readlink(self, path: str) -> str:
         lgr.debug("readlink(path=%r)", path)
         return os.readlink(path)
 
-    # ??? seek seems to be not implemented by fusepy/ Operations
+    # ??? seek seems to be not implemented by mfusepy/ Operations
 
     #
     # Benign writeable operations which we can allow

datalad_fuse/tests/test_fuse.py

@@ -1,6 +1,9 @@
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from contextlib import contextmanager
+from errno import ENOENT
 import hashlib
+import inspect
+import os
 import os.path
 from pathlib import Path
 import subprocess
@@ -241,6 +244,39 @@ def test_fuse_git_status(tmp_path):
         assert r.stdout == ""
 
 
+@pytest.mark.parametrize("transparent", [False, True])
+def test_callbacks_map_paths(tmp_path, transparent):
+    # In-process check (no mount) that callbacks, as mfusepy invokes them,
+    # get the FUSE path mapped into the dataset and .git hidden as needed.
+    from mfusepy import FuseOSError
+
+    from datalad_fuse.fuse_ import DataLadFUSE
+
+    ds = Dataset(tmp_path / "ds").create(cfg_proc="text2git")
+    (tmp_path / "ds" / "text.txt").write_text("text\n")
+    ds.save(message="Create text file")
+    fs = DataLadFUSE(ds.path, caching=False, mode_transparent=transparent)
+    assert "text.txt" in fs.readdir("/", 0)
+    assert fs.getattr("/text.txt")["st_size"] == 5
+    fh = fs.open("/text.txt", os.O_RDONLY)
+    try:
+        assert fs.read("/text.txt", 100, 0, fh) == b"text\n"
+    finally:
+        fs.release("/text.txt", fh)
+    if transparent:
+        assert ".git" in fs.readdir("/", 0)
+        assert fs.getattr("/.git/HEAD")["st_size"] > 0
+    else:
+        assert ".git" not in fs.readdir("/", 0)
+        with pytest.raises(FuseOSError) as excinfo:
+            fs.getattr("/.git/HEAD")
+        assert excinfo.value.errno == ENOENT
+    # mfusepy passes `flags` to create() unless it takes just (path, mode)
+    assert list(inspect.signature(fs.create).parameters) == ["path", "mode"]
+    # callbacks we do not implement are left to libfuse
+    assert getattr(fs.access, "libfuse_ignore", False)
+
+
 def test_parallel_access(tmp_path, big_url_dataset):
     ds, data_files = big_url_dataset
     with fusing(ds.path, tmp_path) as mount:

docs/source/conf.py

@@ -79,11 +79,11 @@
 autodoc_member_order = "bysource"
 autodoc_typehints = "description"
 autodoc_default_options = {"members": True, "show-inheritance": True}
-# fusepy needs libfuse at import time; mock it if libfuse is not installed
+# mfusepy needs libfuse at import time; mock it if libfuse is not installed
 try:
-    import fuse  # noqa: F401
+    import mfusepy  # noqa: F401
 except (ImportError, OSError):
-    autodoc_mock_imports = ["fuse"]
+    autodoc_mock_imports = ["mfusepy"]
 
 napoleon_google_docstring = False
 napoleon_numpy_docstring = True

docs/source/installation.rst

@@ -13,26 +13,15 @@ Requirements
   <https://handbook.datalad.org/en/latest/intro/installation.html>`_ covers
   more options.
 - **FUSE**, only for mounting datasets with ``datalad fusefs``.  The
-  `fusepy <https://github.com/fusepy/fusepy>`_ bindings used by
-  ``datalad-fuse`` need the FUSE 2 library (``libfuse.so.2``), plus the
-  ``fusermount`` helper, which FUSE 3 also provides.  Recent Debian and
-  Ubuntu releases come with FUSE 3 only; add the FUSE 2 library to it, keeping
-  FUSE 3 installed:
+  `mfusepy <https://github.com/mxmlnkn/mfusepy>`_ bindings used by
+  ``datalad-fuse`` work with the FUSE 3 or the FUSE 2 library, together with
+  its ``fusermount3`` or ``fusermount`` helper.  On Debian and Ubuntu::
 
-  - Debian 13 (trixie), Ubuntu 24.04 and newer::
+        sudo apt-get install fuse3
 
-        sudo apt-get install fuse3 libfuse2t64
-
-  - Debian 12 (bookworm), Ubuntu 22.04::
-
-        sudo apt-get install fuse3 libfuse2
-
-  - older releases::
-
-        sudo apt-get install fuse
-
-  On these releases, installing the ``fuse`` package instead would remove
-  ``fuse3``, and with it possibly other packages that depend on it.
+  (``sudo apt-get install fuse`` on releases without the ``fuse3`` package).
+  If both libraries are installed, FUSE 2 is used; set the environment
+  variable ``FUSE_LIBRARY_NAME=fuse3`` to use FUSE 3 instead.
 
   FUSE mounts are developed and tested on Linux.  Opening files from Python
   (see :doc:`python`) and ``datalad fsspec-head`` do not need FUSE at all.

docs/source/python.rst

@@ -230,15 +230,15 @@ where :mod:`multiprocessing` starts new processes by re-importing the main
 module, which is the default on macOS and, from Python 3.14 on, on Linux.
 
 To pass other FUSE mount options, mount the file system class
-`~datalad_fuse.fuse_.DataLadFUSE` directly with fusepy; keyword arguments of
+`~datalad_fuse.fuse_.DataLadFUSE` directly with mfusepy; keyword arguments of
 ``FUSE()`` become mount options.  ``DataLadFUSE`` needs the absolute path of
 the dataset, without symbolic links, as returned by :func:`os.path.realpath`:
 
 .. code-block:: python
 
    import os
 
-   from fuse import FUSE
+   from mfusepy import FUSE
 
    from datalad_fuse.fuse_ import DataLadFUSE
 

setup.cfg

@@ -17,8 +17,8 @@ install_requires =
     aiohttp-retry ~= 2.8
     boto3
     datalad >= 0.17.0
-    fsspec[fuse,http] >= 2022.1.0, != 2022.10.0
-    fusepy
+    fsspec[http] >= 2022.1.0, != 2022.10.0
+    mfusepy ~= 3.1
     methodtools ~= 0.4.5
     typing_extensions; python_version < '3.10'
 packages = find_namespace:

tox.ini

@@ -19,6 +19,7 @@ python =
 passenv =
     HOME
     DATALAD_*
+    FUSE_LIBRARY_NAME
 extras = test
 # Factors:
 #   nonetwork — disable network tests (sets bogus proxies to catch leaks)