From e6739cf2270c11b90d72ca38aad0ee2bc17a5e73 Mon Sep 17 00:00:00 2001 From: Leif Denby Date: Thu, 5 Mar 2026 15:28:42 +0100 Subject: [PATCH 1/3] warning and cleanup in zarr_format checks add warning to report when validating remote datasets and storage_options are missing in dataset attributes --- .../checks/global_attributes/zarr_format.py | 167 +++++++++++++----- 1 file changed, 123 insertions(+), 44 deletions(-) diff --git a/mlcast_dataset_validator/checks/global_attributes/zarr_format.py b/mlcast_dataset_validator/checks/global_attributes/zarr_format.py index cb9e8fd..875a96f 100644 --- a/mlcast_dataset_validator/checks/global_attributes/zarr_format.py +++ b/mlcast_dataset_validator/checks/global_attributes/zarr_format.py @@ -9,31 +9,24 @@ SECTION_ID = f"{PARENT_SECTION_ID}.3" -def has_consolidated_metadata(ds, storage_options=None): +def _has_consolidated_metadata(store_path: str, storage_options=None): """ Check whether a Zarr dataset opened via xarray has consolidated metadata. Parameters ---------- - ds : xarray.Dataset - The dataset opened with `xr.open_zarr()`. + store_path : str + Path/URL to the backing Zarr store. storage_options : dict, optional The same storage_options that were used when opening the dataset. Required for remote stores (e.g. S3, GCS). Returns ------- - bool or None - True if `.zmetadata` exists, - False if not found, - None if source path cannot be determined. - """ - # Try to infer the original store path (xarray stores this in encoding) - store_path = ds.encoding.get("source") - if store_path is None: - return None # no source info (e.g. dataset from memory) + bool + ``True`` if `.zmetadata` exists at the store root, otherwise ``False``. - # Create filesystem using same storage options as xarray + """ fs, _, paths = fsspec.get_fs_token_paths( store_path, storage_options=storage_options ) @@ -41,20 +34,26 @@ def has_consolidated_metadata(ds, storage_options=None): return fs.exists(f"{store_root}/.zmetadata") -def _detect_zarr_format(ds, storage_options=None): - """Detect Zarr format version from the store on disk. - - Zarr v3 stores have a ``zarr.json`` file at the root, while v2 stores - have ``.zgroup``. xarray does not expose the format version as a - dataset attribute, so we inspect the store directly. +def _detect_zarr_format(store_path: str, storage_options=None): """ - store_path = ds.encoding.get("source") - if store_path is None: - return 2 # cannot determine, assume v2 + Detect Zarr format version from the backing store. - if storage_options is None: - storage_options = ds.encoding.get("storage_options") + Parameters + ---------- + store_path : str + Path/URL to the backing Zarr store. + storage_options : dict, optional + Filesystem options used to access the store (for example S3/GCS + credentials or endpoint settings). If omitted, fsspec defaults are used. + + Returns + ------- + int + Detected Zarr format version: + - ``3`` when ``zarr.json`` exists at the store root. + - ``2`` otherwise. + """ fs, _, paths = fsspec.get_fs_token_paths( store_path, storage_options=storage_options ) @@ -65,50 +64,130 @@ def _detect_zarr_format(ds, storage_options=None): return 2 +def _is_remote_store(store_path: str) -> bool: + """ + Determine whether a store path refers to a remote filesystem. + + Parameters + ---------- + store_path : str + Path/URL to the backing Zarr store. + + Returns + ------- + bool + ``True`` when the inferred protocol is not local file-based, otherwise ``False``. + """ + protocol = fsspec.utils.infer_storage_options(store_path).get("protocol") + return protocol not in (None, "file", "local") + + @log_function_call def check_zarr_format( ds: xr.Dataset, *, - storage_options: dict = None, allowed_versions: Sequence[int], require_consolidated_if_v2: bool, ) -> ValidationReport: - """Check Zarr format requirements.""" - report = ValidationReport() + """ + Validate Zarr format compatibility and consolidated metadata requirements. - if storage_options is None: - storage_options = ds.encoding.get("storage_options") + Parameters + ---------- + ds : xarray.Dataset + Dataset under validation. + allowed_versions : Sequence[int] + Zarr format versions accepted by the specification. + require_consolidated_if_v2 : bool + Whether Zarr v2 datasets must include consolidated metadata. - zarr_format = _detect_zarr_format(ds, storage_options) - if zarr_format in allowed_versions: - report.add( - SECTION_ID, - "Zarr version compatibility", - "PASS", - f"Using supported Zarr v{zarr_format} format", - ) - else: + Returns + ------- + ValidationReport + Report containing PASS/FAIL/WARNING entries for Zarr version checks and, + when applicable, consolidated metadata validation. + """ + report = ValidationReport() + + store_path = ds.encoding.get("source") + if not store_path: report.add( SECTION_ID, "Zarr version compatibility", "FAIL", - f"Unsupported Zarr version: v{zarr_format}", + "Cannot determine Zarr version because dataset source path is missing.", ) - - if zarr_format == 2 and require_consolidated_if_v2: - if has_consolidated_metadata(ds, storage_options=storage_options): + if require_consolidated_if_v2: report.add( SECTION_ID, "Consolidated metadata presence", + "FAIL", + "Cannot verify consolidated metadata because dataset source path is missing.", + ) + return report + + storage_options = ds.encoding.get("storage_options") + try: + zarr_format = _detect_zarr_format(store_path, storage_options) + if zarr_format in allowed_versions: + report.add( + SECTION_ID, + "Zarr version compatibility", "PASS", - "Zarr v2 dataset has consolidated metadata", + f"Using supported Zarr v{zarr_format} format", ) else: + report.add( + SECTION_ID, + "Zarr version compatibility", + "FAIL", + f"Unsupported Zarr version: v{zarr_format}", + ) + + if zarr_format == 2 and require_consolidated_if_v2: + consolidated = _has_consolidated_metadata( + store_path, storage_options=storage_options + ) + if consolidated is True: + report.add( + SECTION_ID, + "Consolidated metadata presence", + "PASS", + "Zarr v2 dataset has consolidated metadata", + ) + else: + report.add( + SECTION_ID, + "Consolidated metadata presence", + "FAIL", + "Zarr v2 dataset is missing consolidated metadata", + ) + except FileNotFoundError as exc: + if _is_remote_store(store_path): + report.add( + SECTION_ID, + "Zarr store accessibility", + "WARNING", + ( + "Could not access dataset store for Zarr checks. " + "Since this appears to be a remote store, your may need to " + "set any `storage_options` that you provided to xr.open_zarr(...) " + "as ds.encoding['storage_options'] so that validation checks can access the store. " + f"({exc})" + ), + ) + report.add( + SECTION_ID, + "Zarr version compatibility", + "FAIL", + "Cannot determine Zarr version because dataset store is not accessible.", + ) + if require_consolidated_if_v2: report.add( SECTION_ID, "Consolidated metadata presence", "FAIL", - "Zarr v2 dataset is missing consolidated metadata", + "Cannot verify consolidated metadata because dataset store is not accessible.", ) return report From 5fe6e8d68170b683ce856d74e8c01c1c480025ac Mon Sep 17 00:00:00 2001 From: Leif Denby Date: Sun, 8 Mar 2026 21:11:10 +0000 Subject: [PATCH 2/3] catch S3 exception for missing anon=True --- .../checks/global_attributes/zarr_format.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mlcast_dataset_validator/checks/global_attributes/zarr_format.py b/mlcast_dataset_validator/checks/global_attributes/zarr_format.py index 875a96f..3147dfd 100644 --- a/mlcast_dataset_validator/checks/global_attributes/zarr_format.py +++ b/mlcast_dataset_validator/checks/global_attributes/zarr_format.py @@ -2,6 +2,7 @@ import fsspec import xarray as xr +from botocore.exceptions import NoCredentialsError from ...specs.reporting import ValidationReport, log_function_call from . import SECTION_ID as PARENT_SECTION_ID @@ -162,7 +163,10 @@ def check_zarr_format( "FAIL", "Zarr v2 dataset is missing consolidated metadata", ) - except FileNotFoundError as exc: + # not providing anon=True in storage_options for an S3 store without + # credentials will raise a NoCredentialsError, while a local store that + # doesn't exist will raise a FileNotFoundError + except (FileNotFoundError, NoCredentialsError) as exc: if _is_remote_store(store_path): report.add( SECTION_ID, From e3792f05c5f6bdc09ef72868fe62e1b2d624a9c4 Mon Sep 17 00:00:00 2001 From: Leif Denby Date: Sun, 8 Mar 2026 21:12:47 +0000 Subject: [PATCH 3/3] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 704b1bd..a14f200 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Make checks on max spatial resolution (1km) more lenient using math.isclose [\#30](https://github.com/mlcast-community/mlcast-dataset-validator/pull/30), @ladc - Detect Zarr v3 format from store files (`zarr.json`) instead of relying on `getattr(ds, "zarr_format", 2)` which always defaulted to v2, causing v3 stores to incorrectly fail the consolidated metadata check [\#27](https://github.com/mlcast-community/mlcast-dataset-validator/pull/27), @franchg - Fix for package version in ci build of html render of specs [\#25](https://github.com/mlcast-community/mlcast-dataset-validator/pull/25), @leifdenby +- Ensure zarr format checks fail if requirements cannot be validated due to missing access to underlying zarr store [\#31](https://github.com/mlcast-community/mlcast-dataset-validator/pull/31), @leifdenby ## [v0.2.0](https://github.com/mlcast-community/mlcast-dataset-validator/releases/tag/v0.2.0)