Memoize readable fingerprints

dandi/metadata/nwb.py

@@ -21,17 +21,18 @@
     ignore_benign_pynwb_warnings,
     metadata_cache,
     nwb_has_external_links,
+    readable_fingerprint,
 )
 from ..utils import find_parent_directory_containing
 
 lgr = get_logger()
 
 
 # Disable this for clean hacking
-@metadata_cache.memoize_path
+@metadata_cache.memoize_path(custom_fingerprint=readable_fingerprint)
 def get_metadata(
     path: str | Path | Readable, digest: Digest | None = None
-) -> dict | None:
+) -> dict[str, Any]:
     """
     Get "flatdata" from a .nwb file
 

dandi/misctypes.py

@@ -284,6 +284,25 @@ def get_filename(self) -> str:
         """
         ...
 
+    def get_fingerprint(self) -> str | None:
+        """
+        .. versionadded:: 0.81.0
+
+        Returns a fingerprint of the resource's own content, such as a content
+        digest (e.g., a git-annex key), or `None` if none is known
+
+        Two resources with equal fingerprints must have identical bytes.  The
+        fingerprint covers that one file only, not its location nor any other
+        file that may affect how it is interpreted, such as the BIDS sidecar
+        ``.json`` files (possibly inherited from parent directories) describing
+        a ``.nii.gz``.  Results computed from the file alone may thus be
+        shared by resources with equal fingerprints (see
+        `dandi.pynwb_utils.readable_fingerprint`, which also pairs it with the
+        file name); results that depend on other files must not be keyed by
+        it alone.
+        """
+        return None
+
 
 class LocalReadableFile(Readable):
     """
@@ -343,9 +362,8 @@ class RemoteReadableAsset(Readable):
 
     def open(self) -> IO[bytes]:
         # Optional dependency:
-        import fsspec
-
         from aiohttp import ClientTimeout
+        import fsspec
 
         # We need to call open() on the return value of fsspec.open() because
         # otherwise the filehandle will only be opened when used to enter a

dandi/pynwb_utils.py

@@ -73,6 +73,27 @@
 )
 
 
+def readable_fingerprint(source: Any) -> tuple[str, str] | None:
+    """
+    Content fingerprint of ``source`` for ``PersistentCache.memoize_path``
+
+    Pass it as ``custom_fingerprint`` to cache the results of a function of a
+    local path or a `Readable`.  Paths keep being cached under their location
+    and ``stat()``; a `Readable` cannot be fingerprinted that way, so one whose
+    `~Readable.get_fingerprint` returns a value is cached under its file name
+    and that fingerprint instead.  A `Readable` without a fingerprint is handled
+    as without this: cached by its path if it is path-like (as
+    `LocalReadableFile` is), not cached at all otherwise.
+
+    Only for functions whose result depends on the file's own content and name
+    (see `~Readable.get_fingerprint`): those it is applied to here each read a
+    single NWB file.
+    """
+    if isinstance(source, Readable) and (fp := source.get_fingerprint()) is not None:
+        return (source.get_filename(), fp)
+    return None
+
+
 def _sanitize_nwb_version(
     v: Any,
     filename: str | Path | None = None,
@@ -194,7 +215,7 @@ def get_neurodata_types_to_modalities_map() -> dict[str, str]:
     return ndtypes
 
 
-@metadata_cache.memoize_path
+@metadata_cache.memoize_path(custom_fingerprint=readable_fingerprint)
 def get_neurodata_types(filepath: str | Path | Readable) -> list[str]:
     with open_readable(filepath) as fp, h5py.File(fp, "r") as h5file:
         all_pairs = _scan_neurodata_types(h5file)
@@ -808,7 +829,7 @@ def copy_nwb_file(src: str | Path, dest: str | Path) -> str:
     return str(dest)
 
 
-@metadata_cache.memoize_path
+@metadata_cache.memoize_path(custom_fingerprint=readable_fingerprint)
 def nwb_has_external_links(filepath: str | Path | Readable) -> bool:
     with open_readable(filepath) as f, h5py.File(f, "r") as fp:
         visited = set()