|
12 | 12 | from __future__ import annotations |
13 | 13 |
|
14 | 14 | from collections import Counter |
15 | | -from collections.abc import Callable, Sequence |
| 15 | +from collections.abc import Callable |
16 | 16 | from datetime import timedelta |
17 | | -from functools import wraps |
18 | 17 | import inspect |
19 | 18 | import os |
20 | 19 | import os.path as op |
|
64 | 63 | get_module_version(hdmf), |
65 | 64 | get_module_version(h5py), |
66 | 65 | ] |
67 | | -metadata_cache_tokens = dandi_cache_tokens |
68 | 66 | metadata_cache = PersistentCache( |
69 | | - name="dandi-metadata", tokens=metadata_cache_tokens, envvar="DANDI_CACHE" |
| 67 | + name="dandi-metadata", tokens=dandi_cache_tokens, envvar="DANDI_CACHE" |
70 | 68 | ) |
71 | | -validate_cache_tokens = dandi_cache_tokens + [get_module_version(dandischema)] |
72 | 69 | validate_cache = PersistentCache( |
73 | | - name="dandi-validate", tokens=validate_cache_tokens, envvar="DANDI_CACHE" |
| 70 | + name="dandi-validate", |
| 71 | + tokens=dandi_cache_tokens + [get_module_version(dandischema)], |
| 72 | + envvar="DANDI_CACHE", |
74 | 73 | ) |
75 | 74 |
|
76 | | -F = TypeVar("F", bound=Callable[..., Any]) |
77 | 75 |
|
78 | | - |
79 | | -def memoize_source(cache: PersistentCache, tokens: Sequence[Any]) -> Callable[[F], F]: |
| 76 | +def readable_fingerprint(source: Any) -> tuple[str, str] | None: |
80 | 77 | """ |
81 | | - Decorator caching the results of a function of a local path or a `Readable` |
82 | | -
|
83 | | - For a path argument (`str` or `Path`), this is exactly ``cache.memoize_path``: |
84 | | - the result is cached under the file's location, a fingerprint of the file |
85 | | - (mtime, ctime, size, ...), and ``tokens``. A `Readable` cannot be |
86 | | - fingerprinted that way, so `PersistentCache.memoize_path` leaves calls on |
87 | | - one uncached; here, a `Readable` whose `~Readable.get_fingerprint` returns |
88 | | - a value is cached under its file name, that fingerprint, and ``tokens`` |
89 | | - instead. A `Readable` without a fingerprint is still never cached. |
90 | | -
|
91 | | - ``tokens`` should be the ones ``cache`` was created with (versions of the |
92 | | - software producing the results): `PersistentCache.memoize`, unlike |
93 | | - `~PersistentCache.memoize_path`, does not add them by itself. |
| 78 | + Content fingerprint of ``source`` for ``PersistentCache.memoize_path`` |
| 79 | +
|
| 80 | + Pass it as ``content_fingerprint`` to cache the results of a function of a |
| 81 | + local path or a `Readable`. Paths keep being cached under their location |
| 82 | + and ``stat()``; a `Readable` cannot be fingerprinted that way, so one whose |
| 83 | + `~Readable.get_fingerprint` returns a value is cached under its file name |
| 84 | + and that fingerprint instead. A `Readable` without a fingerprint is handled |
| 85 | + as without this: cached by its path if it is path-like (as |
| 86 | + `LocalReadableFile` is), not cached at all otherwise. |
94 | 87 | """ |
95 | | - |
96 | | - def decorator(f: F) -> F: |
97 | | - sig = inspect.signature(f) |
98 | | - if any(p.kind is p.VAR_POSITIONAL for p in sig.parameters.values()): |
99 | | - raise TypeError(f"memoize_source: {f.__qualname__}() must not take *args") |
100 | | - by_path = cache.memoize_path(f) |
101 | | - |
102 | | - def by_fingerprint( |
103 | | - filename: str, |
104 | | - fingerprint: str, |
105 | | - cache_tokens: tuple[Any, ...], |
106 | | - *, |
107 | | - _source: Readable, |
108 | | - **kwargs: Any, |
109 | | - ) -> Any: |
110 | | - return f(_source, **kwargs) |
111 | | - |
112 | | - # Give it an identity of its own under `f`'s module, so that joblib |
113 | | - # keeps its results in a per-function directory next to `by_path`'s |
114 | | - by_fingerprint.__module__ = f.__module__ |
115 | | - by_fingerprint.__name__ = f"{f.__name__}__by_fingerprint" |
116 | | - by_fingerprint.__qualname__ = f"{f.__qualname__}__by_fingerprint" |
117 | | - # fscacher >= 0.4 calls the argument `exclude_kwargs`, older ones `ignore` |
118 | | - ignore_kwarg = ( |
119 | | - "exclude_kwargs" |
120 | | - if "exclude_kwargs" in inspect.signature(cache.memoize).parameters |
121 | | - else "ignore" |
122 | | - ) |
123 | | - cached_by_fingerprint = cache.memoize( |
124 | | - by_fingerprint, **{ignore_kwarg: ["_source"]} |
125 | | - ) |
126 | | - |
127 | | - @wraps(f) |
128 | | - def wrapper(source: Any, *args: Any, **kwargs: Any) -> Any: |
129 | | - if not isinstance(source, Readable): |
130 | | - return by_path(source, *args, **kwargs) |
131 | | - fingerprint = source.get_fingerprint() |
132 | | - if fingerprint is None: |
133 | | - return f(source, *args, **kwargs) |
134 | | - # Pass `f`'s other arguments by name (defaults included), so that the |
135 | | - # cache key does not depend on how they were passed, and so that |
136 | | - # joblib < 1.4 does not mistake positional ones for `_source` |
137 | | - bound = sig.bind(source, *args, **kwargs) |
138 | | - bound.apply_defaults() |
139 | | - others: dict[str, Any] = {} |
140 | | - for name, value in list(bound.arguments.items())[1:]: |
141 | | - if sig.parameters[name].kind is inspect.Parameter.VAR_KEYWORD: |
142 | | - others.update(value) |
143 | | - else: |
144 | | - others[name] = value |
145 | | - return cached_by_fingerprint( |
146 | | - source.get_filename(), |
147 | | - fingerprint, |
148 | | - tuple(tokens), |
149 | | - _source=source, |
150 | | - **others, |
151 | | - ) |
152 | | - |
153 | | - return cast(F, wrapper) |
154 | | - |
155 | | - return decorator |
| 88 | + if isinstance(source, Readable) and (fp := source.get_fingerprint()) is not None: |
| 89 | + return (source.get_filename(), fp) |
| 90 | + return None |
156 | 91 |
|
157 | 92 |
|
158 | 93 | def _sanitize_nwb_version( |
@@ -276,7 +211,7 @@ def get_neurodata_types_to_modalities_map() -> dict[str, str]: |
276 | 211 | return ndtypes |
277 | 212 |
|
278 | 213 |
|
279 | | -@memoize_source(metadata_cache, metadata_cache_tokens) |
| 214 | +@metadata_cache.memoize_path(content_fingerprint=readable_fingerprint) |
280 | 215 | def get_neurodata_types(filepath: str | Path | Readable) -> list[str]: |
281 | 216 | with open_readable(filepath) as fp, h5py.File(fp, "r") as h5file: |
282 | 217 | all_pairs = _scan_neurodata_types(h5file) |
@@ -890,7 +825,7 @@ def copy_nwb_file(src: str | Path, dest: str | Path) -> str: |
890 | 825 | return str(dest) |
891 | 826 |
|
892 | 827 |
|
893 | | -@memoize_source(metadata_cache, metadata_cache_tokens) |
| 828 | +@metadata_cache.memoize_path(content_fingerprint=readable_fingerprint) |
894 | 829 | def nwb_has_external_links(filepath: str | Path | Readable) -> bool: |
895 | 830 | with open_readable(filepath) as f, h5py.File(f, "r") as fp: |
896 | 831 | visited = set() |
|
0 commit comments