From 228026adc1a1e96f5268fd416a1638b6061ca15c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 6 Oct 2026 21:21:00 +0000 Subject: [PATCH] Use datalad-fuse's pluggable backends (remfile) when available With the development version of datalad-fuse from datalad/datalad-fuse#131, use its RemoteFilesystemAdapter, which reads HDF5-based files such as NWB with remfile if installed (fsspec otherwise), and look up S3 exports for content without a registered URL. The released FsspecAdapter is still used with datalad-fuse 0.5.1. Test it in a new datalad-dev CI job. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qn5WBSiQgoZoL4fytF6nEr --- .github/workflows/run-tests.yml | 10 ++++++++++ dandi/support/datalad_fuse.py | 25 +++++++++++++++++++----- dandi/support/tests/test_datalad_fuse.py | 25 ++++++++++++++++++++++++ docs/source/cmdline/validate.rst | 6 ++++++ 4 files changed, 61 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index d33f99a4f..c1129c7f9 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -76,6 +76,9 @@ jobs: - os: ubuntu-latest python: '3.12' mode: datalad + - os: ubuntu-latest + python: '3.12' + mode: datalad-dev steps: - name: Set up environment @@ -130,6 +133,13 @@ jobs: if: matrix.mode == 'datalad' run: pip install git-annex ".[datalad]" + # TODO: drop once datalad/datalad-fuse#131 is released + - name: Install git-annex, remfile and development datalad-fuse + if: matrix.mode == 'datalad-dev' + run: | + pip install git-annex remfile ".[datalad]" \ + "datalad-fuse @ git+https://github.com/datalad/datalad-fuse@refs/pull/131/head" + - name: Create NFS filesystem if: matrix.mode == 'nfs' run: | diff --git a/dandi/support/datalad_fuse.py b/dandi/support/datalad_fuse.py index cfe71d895..a6cea1f49 100644 --- a/dandi/support/datalad_fuse.py +++ b/dandi/support/datalad_fuse.py @@ -28,9 +28,11 @@ from __future__ import annotations import atexit +from collections.abc import Iterator from dataclasses import dataclass, field from datetime import datetime from functools import cache +from itertools import chain import logging import os from pathlib import Path @@ -51,11 +53,17 @@ def get_adapter() -> Any: root so that it serves files of any dataset, and closed at exit """ # Optional dependency: - from datalad_fuse.fsspec import FsspecAdapter + try: + # datalad-fuse with pluggable backends (datalad/datalad-fuse#131): + # remfile for HDF5-based files (such as NWB) if installed, fsspec + # otherwise, as configured with datalad.fusefs.backends + from datalad_fuse.adapter import RemoteFilesystemAdapter as Adapter + except ImportError: + from datalad_fuse.fsspec import FsspecAdapter as Adapter # caching=False: do not keep the streamed blocks on disk, in the dataset. # The root and all paths passed to the adapter must be absolute. - adapter = FsspecAdapter(Path(os.path.abspath(os.sep)), caching=False) + adapter = Adapter(Path(os.path.abspath(os.sep)), caching=False) atexit.register(adapter.__exit__, None, None, None) return adapter @@ -141,9 +149,12 @@ def get_annexed_readable(path: str | Path) -> AnnexedReadableFile | None: # Not a broken symbolic link return None try: - from datalad_fuse.fsspec import FileState + from datalad_fuse.adapter import FileState except ImportError: - return None + try: + from datalad_fuse.fsspec import FileState + except ImportError: + return None if not annex_initialized(filepath.parent): return None try: @@ -155,7 +166,11 @@ def get_annexed_readable(path: str | Path) -> AnnexedReadableFile | None: return None if state is not FileState.NO_CONTENT or key is None or key.size is None: return None - url = next(dsap.get_urls(str(key)), None) + urls: Iterator[str] = dsap.get_urls(str(key)) + if hasattr(dsap, "get_exporttree_urls"): + # Fallback to S3 exports of datalad/datalad-fuse#131 + urls = chain(urls, dsap.get_exporttree_urls(relpath, key)) + url = next(urls, None) if url is None: lgr.debug("%s: git-annex knows no URL for key %s", filepath, key) return None diff --git a/dandi/support/tests/test_datalad_fuse.py b/dandi/support/tests/test_datalad_fuse.py index 6f594eb07..b27fcfb36 100644 --- a/dandi/support/tests/test_datalad_fuse.py +++ b/dandi/support/tests/test_datalad_fuse.py @@ -95,3 +95,28 @@ def test_annexed_readable_cached_by_key( assert neurodata_types(get_annexed_readable(nwb)) == expected assert neurodata_types(nwb) == expected assert server.ranged_requests == requests + + +@pytest.mark.ai_generated +def test_annexed_readable_remfile( + streamed_nwb: tuple[Path, RangeHTTPServer], simple2_nwb: Path +) -> None: + """With datalad/datalad-fuse#131 and remfile, NWB files are read with remfile""" + pytest.importorskip("datalad_fuse.adapter") + pytest.importorskip("remfile") + from datalad_fuse.remfile import RemfileWrapper + + nwb, server = streamed_nwb + r = get_annexed_readable(nwb) + assert r is not None + fp = r.open() + try: + assert isinstance(fp, RemfileWrapper) + with h5py.File(fp, "r") as h5: + assert h5.attrs["nwb_version"] + finally: + fp.close() + assert server.ranged_requests > 0 + assert get_neurodata_types.__wrapped__(r) == get_neurodata_types.__wrapped__( + simple2_nwb + ) diff --git a/docs/source/cmdline/validate.rst b/docs/source/cmdline/validate.rst index 00b76eb97..ed0d8ad26 100644 --- a/docs/source/cmdline/validate.rst +++ b/docs/source/cmdline/validate.rst @@ -122,6 +122,11 @@ Notes: - git-annex must be initialized in the clone, which ``datalad clone`` does (after ``git clone``, run ``git annex init``). +- With the development version of datalad-fuse from + https://github.com/datalad/datalad-fuse/pull/131 and remfile_ installed + (``pip install remfile``), NWB and other HDF5-based files are read with + remfile, which is optimized for HDF5's access pattern, and content exported + to S3 without a URL registered in git-annex can be found too. - datalad-fuse does not use the HTTP proxy environment variables (``HTTPS_PROXY`` etc.), so the URLs must be reachable directly. - Some nwbinspector checks read data arrays (e.g., timestamps), so the amount @@ -145,6 +150,7 @@ Notes: the file and folder names of the annexed files themselves. .. _datalad-fuse: https://github.com/datalad/datalad-fuse +.. _remfile: https://github.com/flatironinstitute/remfile Development Options