Skip to content

Commit d18c337

Browse files
committed
Merge branch 'claude/memoize-readable-fingerprint' into claude/brave-hamilton-ast647
AnnexReadableFile and its tests moved to the base; take its versions. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Qn5WBSiQgoZoL4fytF6nEr
2 parents dc448f6 + 69ee2a4 commit d18c337

2 files changed

Lines changed: 31 additions & 2 deletions

File tree

‎dandi/support/annex.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,10 @@
1212
1313
This module reads that information using only ``git`` (git-annex itself is not
1414
required) and exposes the content of such files as a `Readable` that streams it
15-
over HTTP(S) with fsspec_, so that, e.g., ``dandi validate`` can validate a
16-
DataLad Dandiset without downloading the (possibly terabytes of) data.
15+
over HTTP(S) with fsspec_, so that the files of a DataLad Dandiset can be read
16+
(e.g., for their metadata) without downloading the (possibly terabytes of)
17+
data. Its fingerprint is the key, so that results computed from the content
18+
are cached under it.
1719
1820
.. _git-annex: https://git-annex.branchable.com
1921
.. _DataLad: https://www.datalad.org

‎dandi/support/tests/test_annex.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import threading
1313
from typing import Any
1414

15+
from fscacher import PersistentCache
1516
import h5py
1617
import pytest
1718

@@ -23,6 +24,7 @@
2324
get_annex_readable,
2425
parse_url_log,
2526
)
27+
from ...pynwb_utils import get_neurodata_types, readable_fingerprint
2628
from ...tests.fixtures import (
2729
annex_key_for_file,
2830
create_git_annex_branch,
@@ -321,6 +323,31 @@ def range_http_server(tmp_path: Path) -> Iterator[tuple[RangeHTTPServer, str]]:
321323
thread.join()
322324

323325

326+
@pytest.mark.ai_generated
327+
def test_annex_readable_file_cached_by_key(tmp_path: Path, simple2_nwb: Path) -> None:
328+
pytest.importorskip("fsspec")
329+
cache = PersistentCache(path=tmp_path / "cache")
330+
neurodata_types = cache.memoize_path(custom_fingerprint=readable_fingerprint)(
331+
get_neurodata_types.__wrapped__
332+
)
333+
content = tmp_path / "content.nwb"
334+
shutil.copy(simple2_nwb, content)
335+
key = AnnexKey.parse(annex_key_for_file(content))
336+
expected = get_neurodata_types.__wrapped__(content)
337+
streamed = AnnexReadableFile(
338+
filepath=tmp_path / "ds" / "sub-01.nwb", key=key, urls=[content.as_uri()]
339+
)
340+
assert neurodata_types(streamed) == expected
341+
# A file with the same key and name elsewhere (e.g., in another clone) is
342+
# served from the cache, without its content being read (it could not be)
343+
twin = AnnexReadableFile(
344+
filepath=tmp_path / "clone" / "sub-01.nwb", key=key, urls=[]
345+
)
346+
with pytest.raises(RuntimeError):
347+
twin.open()
348+
assert neurodata_types(twin) == expected
349+
350+
324351
@pytest.mark.ai_generated
325352
def test_annex_readable_file_open_http(
326353
range_http_server: tuple[RangeHTTPServer, str], simple2_nwb: Path, tmp_path: Path

0 commit comments

Comments
 (0)