From 36552870083e9ba98816910128c30065c41aee67 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 6 Oct 2026 20:51:13 +0000 Subject: [PATCH 1/4] validate: stream missing annexed content with datalad-fuse Add a `stream` policy to `--missing-file-content`: the content of annexed files that is not present locally (as in a DataLad clone of a Dandiset) is streamed with datalad-fuse's adapter, so that pynwb and nwbinspector run on it without it being downloaded. pynwb validation results are cached under the files' git-annex keys. This is the validation integration of #1933, on top of the datalad-fuse readable rather than a git-only reader of the git-annex branch. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qn5WBSiQgoZoL4fytF6nEr --- dandi/cli/cmd_validate.py | 9 +- dandi/cli/tests/test_cmd_validate.py | 43 +++++++- dandi/files/bases.py | 82 ++++++++++++-- dandi/files/bids.py | 2 +- dandi/pynwb_utils.py | 51 +++++++-- dandi/validate/_core.py | 135 +++++++++++++++++------ dandi/validate/_types.py | 6 + dandi/validate/tests/test_core.py | 158 ++++++++++++++++++++++++++- docs/source/cmdline/validate.rst | 62 ++++++++++- 9 files changed, 495 insertions(+), 53 deletions(-) diff --git a/dandi/cli/cmd_validate.py b/dandi/cli/cmd_validate.py index 5241c5eb4..ea18707c0 100644 --- a/dandi/cli/cmd_validate.py +++ b/dandi/cli/cmd_validate.py @@ -170,8 +170,13 @@ def _filter_results( "in a datalad dataset without fetched data). 'error' (default) emits a " "concise error per file, 'skip' skips each such file with a warning, " "'only-non-data' skips content-dependent validators but still validates " - "path layout.", - type=click.Choice(["error", "only-non-data", "skip"], case_sensitive=True), + "path layout, 'stream' streams the content of annexed files with " + "datalad-fuse so that content-dependent validators run without the files " + "having to be downloaded (requires git-annex and `pip install " + "'dandi[datalad]'`).", + type=click.Choice( + ["error", "only-non-data", "skip", "stream"], case_sensitive=True + ), default="error", ) @click.option( diff --git a/dandi/cli/tests/test_cmd_validate.py b/dandi/cli/tests/test_cmd_validate.py index 1da5c16ca..aef651740 100644 --- a/dandi/cli/tests/test_cmd_validate.py +++ b/dandi/cli/tests/test_cmd_validate.py @@ -1,6 +1,7 @@ import json from pathlib import Path -from typing import cast +import shutil +from typing import Any, cast from click.testing import CliRunner import pytest @@ -998,6 +999,46 @@ def test_validate_missing_file_content_no_broken_symlinks(tmp_path: Path) -> Non assert "FILE_CONTENT_MISSING" not in r_skip.output +@pytest.mark.ai_generated +def test_validate_missing_file_content_stream( + tmp_path: Path, simple3_nwb: Path, range_http_server: tuple[Any, str] +) -> None: + """--missing-file-content=stream validates annexed content with datalad-fuse.""" + from ...tests.fixtures import make_git_annex_dandiset + from ...tests.skip import skipif + + skipif.no_git_annex() + pytest.importorskip("datalad_fuse.fsspec") + _, base_url = range_http_server + shutil.copy(simple3_nwb, tmp_path / "served" / "content.nwb") + ds = tmp_path / "ds" + make_git_annex_dandiset( + ds, {"sub-001/sub-001.nwb": (simple3_nwb, f"{base_url}/content.nwb")} + ) + out = tmp_path / "out.jsonl" + r = CliRunner().invoke( + validate, + [ + "--missing-file-content", + "stream", + "--min-severity", + "INFO", + "-f", + "json_lines", + "-o", + str(out), + str(ds), + ], + ) + assert "Traceback" not in r.output + ids = [rec.id for rec in load_validation_jsonl([str(out)])] + assert "DANDI.FILE_CONTENT_STREAMED" in ids + assert "DANDI.FILE_CONTENT_MISSING" not in ids + # simple3_nwb lacks a subject_id, which only content-based checks notice + assert "NWBI.check_subject_id_exists" in ids + assert r.exit_code == 1 + + @pytest.mark.ai_generated def test_validate_broken_symlink_path_argument(tmp_path: Path) -> None: """A broken symlink can be given directly as a path to validate. diff --git a/dandi/files/bases.py b/dandi/files/bases.py index db502dcc3..086298e88 100644 --- a/dandi/files/bases.py +++ b/dandi/files/bases.py @@ -10,6 +10,7 @@ from pathlib import Path import re from threading import Lock +import traceback from typing import IO, Any, Generic from xml.etree.ElementTree import fromstring @@ -32,7 +33,7 @@ set_asset_schema_key, ) from dandi.metadata.core import get_default_metadata -from dandi.misctypes import DUMMY_DANDI_ETAG, Digest, LocalReadableFile, P +from dandi.misctypes import DUMMY_DANDI_ETAG, Digest, LocalReadableFile, P, Readable from dandi.utils import post_upload_size_check, pre_upload_size_check, yaml_load from dandi.validate._types import ( ORIGIN_INTERNAL_DANDI, @@ -320,12 +321,22 @@ class LocalFileAsset(LocalAsset): an asset of a Dandiset """ + #: An alternative source for the asset's content, used instead of the file + #: at `filepath` when set. This is used to stream the content of an + #: annexed file (e.g., in a DataLad dataset) that is not present locally; + #: see `dandi.support.datalad_fuse`. + content_source: Readable | None = None + + def _content(self) -> Path | Readable: + """The source to read the asset's content from""" + return self.content_source if self.content_source is not None else self.filepath + def get_metadata( self, digest: Digest | None = None, ignore_errors: bool = True, ) -> BareAsset: - metadata = get_default_metadata(self.filepath, digest=digest) + metadata = get_default_metadata(self._content(), digest=digest) metadata.path = self.path return metadata @@ -515,7 +526,7 @@ def get_metadata( from dandi.metadata.nwb import nwb2asset try: - metadata = nwb2asset(self.filepath, digest=digest) + metadata = nwb2asset(self._content(), digest=digest) except Exception as e: lgr.warning( "Failed to extract NWB metadata from %s: %s: %s", @@ -524,7 +535,7 @@ def get_metadata( str(e), ) if ignore_errors: - metadata = get_default_metadata(self.filepath, digest=digest) + metadata = get_default_metadata(self._content(), digest=digest) else: raise metadata.path = self.path @@ -555,12 +566,16 @@ def get_validation_errors( pass else: # Avoid heavy import by importing within function: - from nwbinspector import Importance, inspect_nwbfile, load_config + from nwbinspector import Importance, load_config # Avoid heavy import by importing within function: from dandi.pynwb_utils import validate as pynwb_validate - errors.extend(pynwb_validate(self.filepath, devel_debug=devel_debug)) + errors.extend( + pynwb_validate( + self.filepath, devel_debug=devel_debug, readable=self.content_source + ) + ) if schema_version is not None: errors.extend( super().get_validation_errors( @@ -576,9 +591,7 @@ def get_validation_errors( validator_version=str(_get_nwb_inspector_version()), ) - for error in inspect_nwbfile( - nwbfile_path=self.filepath, - skip_validate=True, + for error in self._inspect_nwbfile( config=load_config(filepath_or_keyword="dandi"), importance_threshold=Importance.BEST_PRACTICE_VIOLATION, ): @@ -623,6 +636,57 @@ def get_validation_errors( ) return errors + def _inspect_nwbfile(self, **kwargs: Any) -> Iterator[Any]: + """ + Yield the messages from inspecting the file with nwbinspector, reading + the content from `content_source` if it is set (in which case what + `nwbinspector.inspect_nwbfile()` does for a local file is replicated + here for the streamed content) + """ + # Avoid heavy import by importing within function: + from nwbinspector import ( + Importance, + InspectorMessage, + inspect_nwbfile, + inspect_nwbfile_object, + ) + + if self.content_source is None: + yield from inspect_nwbfile( + nwbfile_path=self.filepath, skip_validate=True, **kwargs + ) + return + + # Avoid heavy import by importing within function: + import h5py + from pynwb import NWBHDF5IO + + from dandi.pynwb_utils import open_readable + + try: + with open_readable(self.content_source) as fp, h5py.File( + fp, "r" + ) as h5, NWBHDF5IO(file=h5, mode="r", load_namespaces=True) as io: + nwbfile = io.read() + for message in inspect_nwbfile_object(nwbfile_object=nwbfile, **kwargs): + if message is not None: + message.file_path = str(self.filepath) + yield message + except Exception as e: + # Report the failure the same way inspect_nwbfile() does for a + # local file that PyNWB cannot read + yield InspectorMessage( + message=traceback.format_exc(), + importance=Importance.ERROR, + check_function_name=( + f"During io.read(), an error occurred: " + f"{type(e).__module__}.{type(e).__name__}. " + "This indicates that PyNWB was unable to read the file. " + "See the traceback message for more details." + ), + file_path=str(self.filepath), + ) + class VideoAsset(LocalFileAsset): pass diff --git a/dandi/files/bids.py b/dandi/files/bids.py index 008b085dd..7624c40a3 100644 --- a/dandi/files/bids.py +++ b/dandi/files/bids.py @@ -250,7 +250,7 @@ def get_metadata( ) -> BareAsset: metadata = self.bids_dataset_description.get_asset_metadata(self) start_time = end_time = datetime.now().astimezone() - add_common_metadata(metadata, self.filepath, start_time, end_time, digest) + add_common_metadata(metadata, self._content(), start_time, end_time, digest) metadata.path = self.path return metadata diff --git a/dandi/pynwb_utils.py b/dandi/pynwb_utils.py index da6e4d835..34c454bd0 100644 --- a/dandi/pynwb_utils.py +++ b/dandi/pynwb_utils.py @@ -629,8 +629,9 @@ def rename_if_matched(_name: str, obj: Any) -> None: f.visititems(rename_if_matched) -@validate_cache.memoize_path -def validate(path: str | Path, devel_debug: bool = False) -> list[ValidationResult]: +def validate( + path: str | Path, devel_debug: bool = False, *, readable: Readable | None = None +) -> list[ValidationResult]: """Run validation on a file and return errors In case of an exception being thrown, an error message added to the @@ -639,8 +640,38 @@ def validate(path: str | Path, devel_debug: bool = False) -> list[ValidationResu Parameters ---------- path: str or Path + Path of the file, as reported in the returned results + devel_debug: bool, optional + Whether to re-raise exceptions instead of reporting them as errors + readable: Readable, optional + If given, the file's content is read from this `Readable` instead of from + ``path`` (e.g., to stream the content of an annexed file which is not + present locally); the results are then cached only if it is an + `AnnexedReadableFile` (see `annex_fingerprint`). + """ + source: str | Readable = readable if readable is not None else str(path) + # fscacher is untyped, so the memoized _validate returns Any for mypy + return cast( + "list[ValidationResult]", + _validate(source, str(path), devel_debug=devel_debug), + ) + + +@validate_cache.memoize_path(custom_fingerprint=annex_fingerprint) +def _validate( + source: str | Readable, path: str, devel_debug: bool = False +) -> list[ValidationResult]: + """`validate` proper, with the content source as first argument for caching + + Parameters + ---------- + source: str or Readable + What to read the file's content from: its path or a `Readable` + path: str + Path of the file, as reported in the returned results + devel_debug: bool + Whether to re-raise exceptions instead of reporting them as errors """ - path = str(path) # Might come in as pathlib's PATH errors: list[ValidationResult] = [] # To overcome @@ -652,7 +683,7 @@ def validate(path: str | Path, devel_debug: bool = False) -> list[ValidationResu ) version = None try: - version = get_nwb_version(path, sanitize=False) + version = get_nwb_version(source, sanitize=False) except Exception: # we just will not remove any errors, it is required so should be some pass @@ -698,9 +729,15 @@ def validate(path: str | Path, devel_debug: bool = False) -> list[ValidationResu ) try: - # Validates against the namespaces cached in the file; pynwb falls back - # to its own namespaces if the file has none cached - error_outputs = pynwb.validate(path=path) + # Either way, validates against the namespaces cached in the file; pynwb + # falls back to its own namespaces if the file has none cached + if isinstance(source, Readable): + with open_readable(source) as fp, h5py.File(fp, "r") as h5, NWBHDF5IO( + file=h5, mode="r", load_namespaces=True + ) as reader: + error_outputs = pynwb.validate(io=reader) + else: + error_outputs = pynwb.validate(path=source) except Exception as exc: if devel_debug: raise diff --git a/dandi/validate/_core.py b/dandi/validate/_core.py index 371ce5a30..e3c0e4f33 100644 --- a/dandi/validate/_core.py +++ b/dandi/validate/_core.py @@ -10,9 +10,10 @@ from __future__ import annotations from collections.abc import Iterator +from importlib.util import find_spec import os from pathlib import Path -from typing import Any +import shutil from ._types import ( ORIGIN_VALIDATION_DANDI_LAYOUT, @@ -26,7 +27,8 @@ Validator, ) from ..consts import dandiset_metadata_file -from ..files import find_dandi_files +from ..files import DandiFile, LocalFileAsset, find_dandi_files +from ..support.datalad_fuse import AnnexedReadableFile, get_annexed_readable from ..utils import find_parent_directory_containing BIDS_TO_DANDI = { @@ -155,7 +157,8 @@ def _is_broken_symlink(filepath: Path) -> bool: # BIDS error codes that require reading file content (headers, pixel data). -# When ``only-non-data`` is active these are suppressed for broken-symlink files. +# When ``only-non-data`` or ``stream`` is active these are suppressed for +# broken-symlink files (the BIDS validator cannot stream content). _BIDS_CONTENT_DEPENDENT_CODES = frozenset( { "BIDS.NIFTI_HEADER_UNREADABLE", @@ -181,13 +184,20 @@ def validate( Policy for files whose content is unavailable (e.g. broken symlinks in a datalad dataset without fetched data). ``error`` emits a concise error, ``skip`` skips the file with a warning, ``only-non-data`` skips - content-dependent validators but still validates path layout. + content-dependent validators but still validates path layout, and + ``stream`` streams the content of annexed files with datalad-fuse (see + `dandi.support.datalad_fuse`) so that + content-dependent validators run without the content being present + locally. Yields ------ path, errors errors for a path """ + if missing_file_content == MissingFileContent.stream: + _check_streaming_requirements() + # Archive of unique `ValidationResult` objects obtained through # `DandiFile.get_validation_errors()` # Note: This is needed to hold on to the unique `ValidationResult` objects @@ -214,34 +224,43 @@ def validate( p, dandiset_path=dandiset_path, allow_all=allow_any_path ): # Handle broken symlinks (missing file content) - if _is_broken_symlink(df.filepath): - r = _handle_missing_content(df, missing_file_content) - if r is not None: - r_id = id(r) - if r_id not in df_result_ids: - df_results.append(r) - df_result_ids.add(r_id) - yield r - if missing_file_content in ( - MissingFileContent.skip, - MissingFileContent.error, - ): + is_broken = _is_broken_symlink(df.filepath) + if is_broken: + if missing_file_content == MissingFileContent.stream: + readable = _prepare_streaming(df) + # A file whose content cannot be streamed gets an error + # and is otherwise skipped + skip_file = readable is None + else: + readable = None + skip_file = missing_file_content in ( + MissingFileContent.skip, + MissingFileContent.error, + ) + r = _handle_missing_content(df, missing_file_content, readable) + r_id = id(r) + if r_id not in df_result_ids: + df_results.append(r) + df_result_ids.add(r_id) + yield r + if skip_file: continue - # only-non-data: fall through but pass the flag to validators + # only-non-data & stream: fall through but pass the policy to + # the validators - is_broken = _is_broken_symlink(df.filepath) for r in df.get_validation_errors( schema_version=schema_version, devel_debug=devel_debug, missing_file_content=(missing_file_content if is_broken else None), ): - # For broken-symlink files under only-non-data, suppress - # BIDS errors that require reading file content (e.g. + # For broken-symlink files under only-non-data and stream, + # suppress BIDS errors that require reading file content (e.g. # NIFTI_HEADER_UNREADABLE). The validator ran in full so # real files still get those checks. if ( is_broken - and missing_file_content == MissingFileContent.only_non_data + and missing_file_content + in (MissingFileContent.only_non_data, MissingFileContent.stream) and r.id in _BIDS_CONTENT_DEPENDENT_CODES ): continue @@ -252,21 +271,74 @@ def validate( yield r +def _check_streaming_requirements() -> None: + """Raise an informative error if what is needed for streaming file content + is not installed""" + if find_spec("datalad_fuse") is None: + raise RuntimeError( + "Streaming file content requires datalad-fuse; install it with " + "`pip install 'dandi[datalad]'`" + ) + if shutil.which("git-annex") is None: + raise RuntimeError("Streaming file content requires git-annex") + + +def _prepare_streaming(df: DandiFile) -> AnnexedReadableFile | None: + """ + Set up streaming of the content of the annexed file represented by ``df``, + and return the `Readable` that will be used for it, or `None` if the + content cannot be streamed (see `get_annexed_readable()`) + """ + if not isinstance(df, LocalFileAsset): + return None + readable = get_annexed_readable(df.filepath) + if readable is not None: + df.content_source = readable + return readable + + def _handle_missing_content( - df: Any, + df: DandiFile, policy: MissingFileContent, -) -> ValidationResult | None: + readable: AnnexedReadableFile | None = None, +) -> ValidationResult: """Produce a single :class:`ValidationResult` for a file with missing content. - Returns ``None`` when *policy* is ``only-non-data`` (a warning is not - needed because validation still proceeds on the non-data aspects). + For the ``stream`` policy, ``readable`` is the `Readable` streaming the + file's content, or `None` if the content cannot be streamed. """ - from ..files import DandiFile - - assert isinstance(df, DandiFile) filepath = df.filepath - if policy == MissingFileContent.error: + if policy == MissingFileContent.stream: + if readable is not None: + return ValidationResult( + id="DANDI.FILE_CONTENT_STREAMED", + origin=ORIGIN_VALIDATION_DANDI_LAYOUT, + severity=Severity.INFO, + scope=Scope.FILE, + path=filepath, + dandiset_path=df.dandiset_path, + message=( + f"File content is not present locally (git-annex key " + f"{readable.key}); content-dependent validation streams " + f"it from {readable.url}" + ), + ) + return ValidationResult( + id="DANDI.FILE_CONTENT_MISSING", + origin=ORIGIN_VALIDATION_DANDI_LAYOUT, + severity=Severity.ERROR, + scope=Scope.FILE, + path=filepath, + dandiset_path=df.dandiset_path, + message=( + f"File content is not available (broken symlink: " + f"{filepath} -> {os.readlink(filepath)}) and cannot be " + f"streamed: not an annexed file in a git-annex repository " + f"with a URL known to git-annex." + ), + ) + elif policy == MissingFileContent.error: return ValidationResult( id="DANDI.FILE_CONTENT_MISSING", origin=ORIGIN_VALIDATION_DANDI_LAYOUT, @@ -277,8 +349,9 @@ def _handle_missing_content( message=( f"File content is not available (broken symlink: " f"{filepath} -> {os.readlink(filepath)}). " - f"Use --missing-file-content=skip or " - f"--missing-file-content=only-non-data to handle gracefully." + f"Use --missing-file-content=skip, " + f"--missing-file-content=only-non-data, or " + f"--missing-file-content=stream to handle gracefully." ), ) elif policy == MissingFileContent.skip: diff --git a/dandi/validate/_types.py b/dandi/validate/_types.py index f41c8402e..48b18b45a 100644 --- a/dandi/validate/_types.py +++ b/dandi/validate/_types.py @@ -26,6 +26,12 @@ class MissingFileContent(StrEnum): skip = auto() """Skip the file entirely; emit a WARNING noting that validation was skipped.""" + stream = auto() + """Stream the content of annexed files with datalad-fuse (see + `dandi.support.datalad_fuse`) so that content-dependent validators run + without the content being present locally. An INFO result is emitted + for each streamed file, and an ERROR for each file that cannot be streamed.""" + lgr = logging.getLogger(__name__) diff --git a/dandi/validate/tests/test_core.py b/dandi/validate/tests/test_core.py index 7d331bb61..ab3759542 100644 --- a/dandi/validate/tests/test_core.py +++ b/dandi/validate/tests/test_core.py @@ -1,5 +1,7 @@ import json +import os from pathlib import Path +import shutil from typing import Any import pytest @@ -17,7 +19,14 @@ ) from ... import __version__ from ...consts import dandiset_metadata_file -from ...tests.fixtures import BIDS_TESTDATA_SELECTION +from ...pynwb_utils import validate as pynwb_validate +from ...support.datalad_fuse import get_annexed_readable +from ...tests.fixtures import ( + BIDS_TESTDATA_SELECTION, + RangeHTTPServer, + make_git_annex_dandiset, +) +from ...tests.skip import skipif def test_validate_nwb_error(simple3_nwb: Path) -> None: @@ -310,3 +319,150 @@ def test_validate_broken_symlink_real_file_still_validated(tmp_path: Path) -> No assert ( len(broken_pynwb) == 0 ), f"policy={policy.value}: pynwb should not run on the broken symlink" + + +# ---- Tests for streaming the content of annexed files (missing_file_content=stream) ---- + + +@pytest.fixture +def streamable_dandiset( + range_http_server: tuple[RangeHTTPServer, str], simple3_nwb: Path, tmp_path: Path +) -> tuple[Path, RangeHTTPServer]: + """A git-annex dandiset looking like a DataLad clone without fetched content. + + ``sub-001/sub-001.nwb`` is an annexed copy of *simple3_nwb* whose content + is served by the HTTP server, and ``sub-001/sub-001_video.mp4`` an annexed + video registered at a URL that is never read (only the size recorded in + its key is needed). + """ + skipif.no_git_annex() + pytest.importorskip("datalad_fuse.fsspec") + server, base_url = range_http_server + shutil.copy(simple3_nwb, tmp_path / "served" / "content.nwb") + video = tmp_path / "video.mp4" + video.write_bytes(b"not really a video") + ds = tmp_path / "ds" + make_git_annex_dandiset( + ds, + { + "sub-001/sub-001.nwb": (simple3_nwb, f"{base_url}/content.nwb"), + "sub-001/sub-001_video.mp4": (video, f"{base_url}/never-read.mp4"), + }, + ) + return ds, server + + +def _content_results(results: list[ValidationResult], name: str) -> list[tuple]: + """Path-independent summary of the results for the file called *name*.""" + return sorted( + (r.id, r.severity, r.origin.validator, r.message) + for r in results + if r.path is not None + and r.path.name == name + and r.id != "DANDI.FILE_CONTENT_STREAMED" + ) + + +@pytest.mark.ai_generated +def test_validate_stream( + streamable_dandiset: tuple[Path, RangeHTTPServer], + simple3_nwb: Path, + tmp_path: Path, +) -> None: + """stream policy validates the content of annexed files with datalad-fuse.""" + ds, server = streamable_dandiset + results = list(validate(ds, missing_file_content=MissingFileContent.stream)) + + streamed = [r for r in results if r.id == "DANDI.FILE_CONTENT_STREAMED"] + assert sorted(r.path.name for r in streamed if r.path is not None) == [ + "sub-001.nwb", + "sub-001_video.mp4", + ] + assert all(r.severity == Severity.INFO for r in streamed) + assert not [r for r in results if r.id.startswith("DANDI.FILE_CONTENT_MISSING")] + assert server.ranged_requests > 0 + + # Validating the video only needs the size recorded in its key + assert _content_results(results, "sub-001_video.mp4") == [] + + # Content-dependent validation of the NWB file gives the same results as + # for a regular dandiset containing the file itself + local = tmp_path / "local" + (local / "sub-001").mkdir(parents=True) + shutil.copy(ds / dandiset_metadata_file, local / dandiset_metadata_file) + shutil.copy(simple3_nwb, local / "sub-001" / "sub-001.nwb") + expected = list(validate(local)) + # simple3_nwb lacks a subject_id, which only content-based checks notice + assert any(r.origin.validator == Validator.nwbinspector for r in expected) + assert _content_results(results, "sub-001.nwb") == _content_results( + expected, "sub-001.nwb" + ) + + +@pytest.mark.ai_generated +@pytest.mark.skipif( + os.environ.get("DANDI_CACHE") == "ignore", reason="the validation cache is disabled" +) +def test_validate_stream_cached_by_key( + streamable_dandiset: tuple[Path, RangeHTTPServer], +) -> None: + """pynwb validation results of a streamed file are cached under its annex key.""" + ds, server = streamable_dandiset + nwb = ds / "sub-001" / "sub-001.nwb" + readable = get_annexed_readable(nwb) + assert readable is not None + results = pynwb_validate(nwb, readable=readable) + assert not [r for r in results if r.id == "pynwb.GENERIC"] + requests = server.ranged_requests + assert requests > 0 + # Served from the cache, without the content being read again + assert pynwb_validate(nwb, readable=get_annexed_readable(nwb)) == results + assert server.ranged_requests == requests + + +@pytest.mark.ai_generated +def test_validate_stream_not_streamable(tmp_path: Path) -> None: + """stream policy emits an error for a broken symlink that cannot be streamed.""" + pytest.importorskip("datalad_fuse.fsspec") + skipif.no_git_annex() + ds = _make_dandiset_with_broken_symlink(tmp_path) + results = list(validate(ds, missing_file_content=MissingFileContent.stream)) + errs = [r for r in results if r.id == "DANDI.FILE_CONTENT_MISSING"] + assert len(errs) == 1 + assert errs[0].severity == Severity.ERROR + assert errs[0].message is not None + assert "cannot be streamed" in errs[0].message + assert not [r for r in results if r.id == "DANDI.FILE_CONTENT_STREAMED"] + assert not [ + r + for r in results + if r.origin.validator in (Validator.pynwb, Validator.nwbinspector) + ] + + +@pytest.mark.ai_generated +def test_validate_stream_unreadable_url( + streamable_dandiset: tuple[Path, RangeHTTPServer], tmp_path: Path +) -> None: + """A registered URL that cannot be read yields errors, not an exception.""" + ds, _ = streamable_dandiset + (tmp_path / "served" / "content.nwb").unlink() + results = list(validate(ds, missing_file_content=MissingFileContent.stream)) + nwb = ds / "sub-001" / "sub-001.nwb" + assert [ + r for r in results if r.path == nwb and r.id == "DANDI.FILE_CONTENT_STREAMED" + ] + errs = [r for r in results if r.path == nwb and r.severity == Severity.ERROR] + assert {r.origin.validator for r in errs} == { + Validator.pynwb, + Validator.nwbinspector, + } + + +@pytest.mark.ai_generated +def test_validate_stream_requires_datalad_fuse( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr("dandi.validate._core.find_spec", lambda name: None) + with pytest.raises(RuntimeError, match=r"dandi\[datalad\]"): + list(validate(tmp_path, missing_file_content=MissingFileContent.stream)) diff --git a/docs/source/cmdline/validate.rst b/docs/source/cmdline/validate.rst index 3a07d7e7a..a789f6203 100644 --- a/docs/source/cmdline/validate.rst +++ b/docs/source/cmdline/validate.rst @@ -56,7 +56,7 @@ Options Limit the number of results shown per group (or in total when not grouping); the excess is replaced by a count of omitted results -.. option:: --missing-file-content [error|only-non-data|skip] +.. option:: --missing-file-content [error|only-non-data|skip|stream] How to handle files whose content is unavailable, such as the broken symbolic links of a DataLad_ dataset (a git-annex_ repository) whose @@ -73,6 +73,12 @@ Options Skip content-dependent validators (pynwb, nwbinspector, ...) for each such file but still validate its path layout + ``stream`` + Stream the content of each such file with datalad-fuse_ so that + content-dependent validators run without the file having to be + downloaded; see `Validating DataLad Dandisets Remotely`_ + below. + .. option:: --load Instead of running validation, load previously saved results from the @@ -84,6 +90,60 @@ Options .. _git-annex: https://git-annex.branchable.com +Validating DataLad Dandisets Remotely +------------------------------------- + +Every Dandiset on the DANDI Archive is mirrored as a DataLad dataset at +https://github.com/dandisets (with https://github.com/dandisets/dandisets as +the superdataset containing all of them). In such a dataset, the assets are +annexed files: symbolic links that remain broken until the content is fetched, +which for some Dandisets would mean downloading terabytes of data. The +``stream`` policy of :option:`--missing-file-content` lets ``dandi validate`` +run the content-dependent validators (pynwb and nwbinspector for NWB files) on +such files by streaming their content on demand with datalad-fuse_ (without +mounting anything), which asks git-annex where the content is, reading only +the parts of each file that the validators need. This requires git-annex and +``pip install "dandi[datalad]"``. + +For example, to produce a JSON Lines record of *all* validation results for +a Dandiset:: + + datalad clone https://github.com/dandisets/000003 + dandi validate --missing-file-content=stream --min-severity=INFO \ + --format=json_lines --output=000003.jsonl 000003 + +Each streamed file yields an ``INFO``-level ``DANDI.FILE_CONTENT_STREAMED`` +result naming the URL its content was read from; a file that cannot be streamed +(not an annexed file, or no URL known to git-annex) yields a +``DANDI.FILE_CONTENT_MISSING`` error instead. + +Notes: + +- git-annex must be initialized in the clone, which ``datalad clone`` does + (after ``git clone``, run ``git annex init``). +- Some nwbinspector checks read data arrays (e.g., timestamps), so the amount + of data streamed for a file depends on its content; it is nevertheless + usually a small fraction of the file. +- The pynwb validation results and the metadata of a streamed file are cached + under its git-annex key (a digest of its content), just as those of a local + file are cached under its modification time and size, so re-running the + command skips that work for files whose key has not changed since. As for + local files, set the :envvar:`DANDI_CACHE` environment variable to ``clear`` + to start afresh or to ``ignore`` to bypass the cache. +- Zarr assets are stored as separate subdatasets (https://github.com/dandizarrs) + and are not streamed yet: an uninstalled Zarr subdataset is an empty directory + that is not validated at all. Streaming them is a follow-up for when NWB Zarr + support has matured across the ecosystem. +- The BIDS validator cannot stream content, so BIDS errors that require reading + a file (e.g., unreadable NIfTI headers) are suppressed for annexed files under + the ``stream`` and ``only-non-data`` policies. For NWB datasets, the primary + use case, nothing is lost: everything the BIDS validator needs is either + present (the non-annexed sidecar files, which are kept in git) or encoded in + the file and folder names of the annexed files themselves. + +.. _datalad-fuse: https://github.com/datalad/datalad-fuse + + Development Options ------------------- From 3361ae3a68cde55b7fc676f03077bc13964ec109 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 6 Oct 2026 21:18:18 +0000 Subject: [PATCH 2/4] Document URL order and proxy behavior of stream policy Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qn5WBSiQgoZoL4fytF6nEr --- docs/source/cmdline/validate.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/source/cmdline/validate.rst b/docs/source/cmdline/validate.rst index a789f6203..00b76eb97 100644 --- a/docs/source/cmdline/validate.rst +++ b/docs/source/cmdline/validate.rst @@ -113,7 +113,8 @@ a Dandiset:: --format=json_lines --output=000003.jsonl 000003 Each streamed file yields an ``INFO``-level ``DANDI.FILE_CONTENT_STREAMED`` -result naming the URL its content was read from; a file that cannot be streamed +result naming the first URL known for its content (for a Dandiset, the DANDI +Archive's API download URL, which redirects to S3); a file that cannot be streamed (not an annexed file, or no URL known to git-annex) yields a ``DANDI.FILE_CONTENT_MISSING`` error instead. @@ -121,6 +122,8 @@ Notes: - git-annex must be initialized in the clone, which ``datalad clone`` does (after ``git clone``, run ``git annex init``). +- datalad-fuse does not use the HTTP proxy environment variables + (``HTTPS_PROXY`` etc.), so the URLs must be reachable directly. - Some nwbinspector checks read data arrays (e.g., timestamps), so the amount of data streamed for a file depends on its content; it is nevertheless usually a small fraction of the file. From d18843d73c449559c5c79b66c3bb7b770b98a584 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 01:40:01 +0000 Subject: [PATCH 3/4] TEMP: CI diagnostics for datalad job hang (to be reverted) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qn5WBSiQgoZoL4fytF6nEr --- .github/workflows/run-tests.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 97c9ab948..6e259c273 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -152,7 +152,17 @@ jobs: # Start coverage before pytest, which loads dandi's pytest plugin (and # thus imports most of dandi) before pytest-cov would start it run: | - python -m coverage run -m pytest -s -v -m "not obolibrary" dandi + set +e + python -X faulthandler -m coverage run -m pytest -s -v -m "not obolibrary" dandi 2>&1 | tee "$RUNNER_TEMP/pytest.log" + status=${PIPESTATUS[0]} + if [ "$status" -ne 0 ]; then + # TEMPORARY diagnostics: surface the hang/crash tracebacks as an annotation + grep -n -B3 -A80 -E "Timeout|Fatal Python|Current thread|Thread 0x" "$RUNNER_TEMP/pytest.log" | head -400 > "$RUNNER_TEMP/diag.txt" + grep -E "^dandi/.*::" "$RUNNER_TEMP/pytest.log" | tail -5 >> "$RUNNER_TEMP/diag.txt" + echo "::error title=pytest diagnostics::$(sed -e 's/%/%25/g' -e 's/\r//g' "$RUNNER_TEMP/diag.txt" | sed ':a;N;$!ba;s/\n/%0A/g')" + exit "$status" + fi + set -e python -m coverage combine python -m coverage xml From de6df649448e7f8aec91b1a0013e9d1437255d1b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 02:40:26 +0000 Subject: [PATCH 4/4] Revert "TEMP: CI diagnostics for datalad job hang (to be reverted)" This reverts commit d18843d73c449559c5c79b66c3bb7b770b98a584. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qn5WBSiQgoZoL4fytF6nEr --- .github/workflows/run-tests.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 6e259c273..97c9ab948 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -152,17 +152,7 @@ jobs: # Start coverage before pytest, which loads dandi's pytest plugin (and # thus imports most of dandi) before pytest-cov would start it run: | - set +e - python -X faulthandler -m coverage run -m pytest -s -v -m "not obolibrary" dandi 2>&1 | tee "$RUNNER_TEMP/pytest.log" - status=${PIPESTATUS[0]} - if [ "$status" -ne 0 ]; then - # TEMPORARY diagnostics: surface the hang/crash tracebacks as an annotation - grep -n -B3 -A80 -E "Timeout|Fatal Python|Current thread|Thread 0x" "$RUNNER_TEMP/pytest.log" | head -400 > "$RUNNER_TEMP/diag.txt" - grep -E "^dandi/.*::" "$RUNNER_TEMP/pytest.log" | tail -5 >> "$RUNNER_TEMP/diag.txt" - echo "::error title=pytest diagnostics::$(sed -e 's/%/%25/g' -e 's/\r//g' "$RUNNER_TEMP/diag.txt" | sed ':a;N;$!ba;s/\n/%0A/g')" - exit "$status" - fi - set -e + python -m coverage run -m pytest -s -v -m "not obolibrary" dandi python -m coverage combine python -m coverage xml