diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 97c9ab948..f4d85c381 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -76,6 +76,9 @@ jobs: - os: ubuntu-latest python: '3.12' mode: datalad + - os: ubuntu-latest + python: '3.12' + mode: datalad-dev steps: - name: Set up environment @@ -130,6 +133,13 @@ jobs: if: matrix.mode == 'datalad' run: pip install git-annex ".[datalad]" + # TODO: drop once datalad/datalad-fuse#131 is released + - name: Install git-annex, remfile and development datalad-fuse + if: matrix.mode == 'datalad-dev' + run: | + pip install git-annex remfile ".[datalad]" \ + "datalad-fuse @ git+https://github.com/datalad/datalad-fuse@refs/pull/131/head" + - name: Create NFS filesystem if: matrix.mode == 'nfs' run: | diff --git a/dandi/support/datalad_fuse.py b/dandi/support/datalad_fuse.py index 1dfaf117c..d9d010465 100644 --- a/dandi/support/datalad_fuse.py +++ b/dandi/support/datalad_fuse.py @@ -28,10 +28,12 @@ from __future__ import annotations import atexit +from collections.abc import Iterator from dataclasses import dataclass, field from datetime import datetime from functools import cache import gc +from itertools import chain import logging import os from pathlib import Path @@ -51,11 +53,17 @@ def get_adapter() -> Any: root so that it serves files of any dataset, and closed at exit """ # Optional dependency: - from datalad_fuse.fsspec import FsspecAdapter + try: + # datalad-fuse with pluggable backends (datalad/datalad-fuse#131): + # remfile for HDF5-based files (such as NWB) if installed, fsspec + # otherwise, as configured with datalad.fusefs.backends + from datalad_fuse.adapter import RemoteFilesystemAdapter as Adapter + except ImportError: + from datalad_fuse.fsspec import FsspecAdapter as Adapter # caching=False: do not keep the streamed blocks on disk, in the dataset. # The root and all paths passed to the adapter must be absolute. - adapter = FsspecAdapter(Path(os.path.abspath(os.sep)), caching=False) + adapter = Adapter(Path(os.path.abspath(os.sep)), caching=False) atexit.register(adapter.__exit__, None, None, None) return adapter @@ -201,9 +209,12 @@ def get_annexed_readable(path: str | Path) -> AnnexedReadableFile | None: # Not a broken symbolic link return None try: - from datalad_fuse.fsspec import FileState + from datalad_fuse.adapter import FileState except ImportError: - return None + try: + from datalad_fuse.fsspec import FileState + except ImportError: + return None if not annex_initialized(filepath.parent): return None try: @@ -215,7 +226,11 @@ def get_annexed_readable(path: str | Path) -> AnnexedReadableFile | None: return None if state is not FileState.NO_CONTENT or key is None or key.size is None: return None - url = next(dsap.get_urls(str(key)), None) + urls: Iterator[str] = dsap.get_urls(str(key)) + if hasattr(dsap, "get_exporttree_urls"): + # Fallback to S3 exports of datalad/datalad-fuse#131 + urls = chain(urls, dsap.get_exporttree_urls(relpath, key)) + url = next(urls, None) if url is None: lgr.debug("%s: git-annex knows no URL for key %s", filepath, key) return None diff --git a/dandi/support/tests/test_datalad_fuse.py b/dandi/support/tests/test_datalad_fuse.py index d75b20c32..f3c06f478 100644 --- a/dandi/support/tests/test_datalad_fuse.py +++ b/dandi/support/tests/test_datalad_fuse.py @@ -116,3 +116,29 @@ def test_annexed_readable_cached_by_key( assert neurodata_types(get_annexed_readable(nwb)) == expected assert neurodata_types(nwb) == expected assert server.ranged_requests == requests + + +@pytest.mark.ai_generated +def test_annexed_readable_remfile( + streamed_nwb: tuple[Path, RangeHTTPServer], simple2_nwb: Path +) -> None: + """With datalad/datalad-fuse#131 and remfile, NWB files are read with remfile""" + pytest.importorskip("datalad_fuse.adapter") + pytest.importorskip("remfile") + from datalad_fuse.remfile import RemfileWrapper + + nwb, server = streamed_nwb + r = get_annexed_readable(nwb) + assert r is not None + fp = r.open() + try: + # (wrapped to pause garbage collection while it is open) + assert isinstance(fp._fp, RemfileWrapper) # type: ignore[attr-defined] + with h5py.File(fp, "r") as h5: + assert h5.attrs["nwb_version"] + finally: + fp.close() + assert server.ranged_requests > 0 + assert get_neurodata_types.__wrapped__(r) == get_neurodata_types.__wrapped__( + simple2_nwb + ) diff --git a/docs/source/cmdline/validate.rst b/docs/source/cmdline/validate.rst index 00b76eb97..ed0d8ad26 100644 --- a/docs/source/cmdline/validate.rst +++ b/docs/source/cmdline/validate.rst @@ -122,6 +122,11 @@ Notes: - git-annex must be initialized in the clone, which ``datalad clone`` does (after ``git clone``, run ``git annex init``). +- With the development version of datalad-fuse from + https://github.com/datalad/datalad-fuse/pull/131 and remfile_ installed + (``pip install remfile``), NWB and other HDF5-based files are read with + remfile, which is optimized for HDF5's access pattern, and content exported + to S3 without a URL registered in git-annex can be found too. - datalad-fuse does not use the HTTP proxy environment variables (``HTTPS_PROXY`` etc.), so the URLs must be reachable directly. - Some nwbinspector checks read data arrays (e.g., timestamps), so the amount @@ -145,6 +150,7 @@ Notes: the file and folder names of the annexed files themselves. .. _datalad-fuse: https://github.com/datalad/datalad-fuse +.. _remfile: https://github.com/flatironinstitute/remfile Development Options