Add per-asset dataStandard, HED standard, and extensions to StandardsType

.gitignore

@@ -14,3 +14,5 @@ sandbox/
 venv/
 venvs/
 dandischema/_version.py
+uv.lock
+.cache/

CLAUDE.md

@@ -43,6 +43,51 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
 - Imports organized stdlib / third-party / local, alphabetical within groups (isort handles ordering).
 - All imports at module top — no function-local imports (per global rule), unless guarding an optional dep with `try/except ImportError`.
 
+## Architecture
+- `models.py` — all Pydantic models, rooted at `DandiBaseModel`.  Since the
+  consolidation, `Dandiset(CommonModel)` and `BareAsset(CommonModel)` /
+  `Asset(BareAsset)` are the only top-level record models;
+  `PublishedDandiset`/`PublishedAsset` are aliases, and publish-time rules are
+  gated on `datePublished`.
+- Self-referencing models (`PropertyValue`, `StandardsType`, `BioSample`) need
+  `Model.model_rebuild()` right after their definition.
+- Pre-instantiated `StandardsType` dicts (`nwb_standard`, `bids_standard`,
+  `ome_ngff_standard`, `hed_standard`) are used both by dandi-cli (to populate
+  per-asset `dataStandard`) and by `aggregate_assets_summary()` (heuristic
+  fallback for assets uploaded by older clients).
+- `metadata.py` — `validate()`, `migrate()` (upgrades and downgrades; dandi-cli
+  applies it to all outgoing metadata, Asset and Dandiset alike, to talk to
+  older servers), and
+  `aggregate_assets_summary()` (run by dandi-archive to build
+  `Dandiset.assetsSummary` from asset metadata).
+- `consts.py` — `DANDI_SCHEMA_VERSION`, `ALLOWED_INPUT_SCHEMAS`,
+  `ALLOWED_TARGET_SCHEMAS`.
+- Vendorization: `DANDI_INSTANCE_*`/`DANDI_DOI_PREFIX` env vars must be set
+  **before** importing `dandischema.models`; tests use the
+  `clear_dandischema_modules_and_set_env_vars` fixture for that.
+
+## Schema Change Checklist
+When adding/removing/changing fields of any model:
+1. If a schema version with the old layout was already released (a
+   `schema-X.Y.Z` tag exists for the current `DANDI_SCHEMA_VERSION`), bump
+   `DANDI_SCHEMA_VERSION` in `consts.py` and add the previous version to
+   `ALLOWED_INPUT_SCHEMAS` and (if a downgrade path exists)
+   `ALLOWED_TARGET_SCHEMAS`.
+2. Register a downgrade for new fields in `migrate()`: `SIMPLE_DOWNGRADES` for
+   top-level fields that may only be dropped when empty (user-provided
+   content), `LOSSY_DOWNGRADES` for automatically derived ones that may be
+   dropped even when populated, or custom code for nested changes.  dandi-cli
+   runs `migrate()` on all outgoing metadata (Asset and Dandiset), so a field
+   it populates must be downgradable or new clients cannot talk to older
+   servers.
+3. Update `test_duplicate_classes` in `tests/test_models.py` if the same field
+   name now lives on two unrelated models (e.g. `dandi:dataStandard` on
+   `Asset` and `AssetsSummary`).
+4. Add tests for validation, migration, and aggregation of the new field.
+5. Coordinate with dandi-cli: fields it populates need a feature guard there
+   (e.g. `"field" in Model.model_fields`) until its minimum `dandischema`
+   requirement includes them.
+
 ## Schema / Data Files
 - `dandischema/*.yaml` (e.g. `standards.yaml`, `model_standards.yaml`, `personinfo.yaml`) are tracked schema/data inputs — don't reformat or rename casually; downstream consumers and `tools/pubschemata.py` rely on them.
 - `dandischema/_version.py` is generated by `versioningit` (see `[tool.versioningit.write]`); don't edit by hand.

dandischema/consts.py

@@ -1,6 +1,6 @@
 from packaging.version import Version as _Version
 
-DANDI_SCHEMA_VERSION = "0.8.0"
+DANDI_SCHEMA_VERSION = "0.9.0"
 ALLOWED_INPUT_SCHEMAS = [
     "0.4.4",
     "0.5.1",
@@ -17,12 +17,13 @@
     "0.6.9",
     "0.6.10",
     "0.7.0",
+    "0.8.0",
     DANDI_SCHEMA_VERSION,
 ]
 
 # We establish migrations (back) to only a few recent versions.
 # When adding changes, please consider whether a migration path should be added.
-ALLOWED_TARGET_SCHEMAS = ["0.6.10", "0.7.0", DANDI_SCHEMA_VERSION]
+ALLOWED_TARGET_SCHEMAS = ["0.6.10", "0.7.0", "0.8.0", DANDI_SCHEMA_VERSION]
 
 # This allows multiple schemas for validation, whereas target schemas focus on
 # migration.

dandischema/metadata.py

@@ -4,7 +4,7 @@
 from inspect import isclass
 import json
 from pathlib import Path
-from typing import Any, Dict, Iterable, Optional, TypeVar, Union, cast, get_args
+from typing import Any, Dict, Iterable, List, Optional, TypeVar, Union, cast, get_args
 
 from jsonschema.protocols import Validator as JsonschemaValidator
 import pydantic
@@ -306,19 +306,27 @@ def migrate(
     skip_validation: bool = False,
 ) -> dict:
     """
-    Migrate a Dandiset metadata instance to a newer DANDI schema version
+    Migrate a metadata instance to another (newer or older) DANDI schema version
 
-    :param obj: The Dandiset metadata instance to migrate
+    Upgrades are supported for Dandiset metadata.  Downgrades (to one of
+    `ALLOWED_TARGET_SCHEMAS`) remove fields introduced after the target
+    version, and also apply to Asset metadata (as used by dandi-cli to send
+    metadata to servers with an older schema): fields with user-provided
+    content are removed only if empty, while automatically derived ones (such
+    as per-asset ``dataStandard``) are removed regardless.
+
+    :param obj: The metadata instance to migrate
     :param to_version: The target DANDI schema version to migrate to
     :param skip_validation: If True, the given instance will not be validated before
-        migration. Otherwise, the instance will be validated before migration.
-    :return: The Dandiset metadata instance migrated to the target DANDI schema version
+        migration. Otherwise, the instance will be validated (as a Dandiset)
+        before migration.
+    :return: The metadata instance migrated to the target DANDI schema version
     :raises ValueError: If the provided instance doesn't have a `schemaVersion` field
         that specifies a valid DANDI schema version
     :raises ValueError: If `to_version` is not a valid DANDI schema version or an
         allowed target schema
-    :raises ValueError: If the target DANDI schema version is lower than the DANDI
-        schema version of the provided instance
+    :raises ValueError: If downgrading would lose a non-empty value of a field
+        with user-provided content
     """
 
     # --------------------------------------------------------------
@@ -439,6 +447,34 @@ def migrate(
                             f"{obj_version} with {field}={value!r} present"
                         )
 
+    # Downgrades that drop fields even if populated, since their values are
+    # derived automatically (e.g. extracted from data files by clients such as
+    # dandi-cli), so no user-provided information is lost.
+    LOSSY_DOWNGRADES = [
+        # version added, fields to remove
+        ("0.9.0", ["dataStandard"]),  # per-asset dataStandard
+    ]
+    for ver_added, fields in LOSSY_DOWNGRADES:
+        if to_version_tuple < version2tuple(ver_added) <= obj_version_tuple:
+            for field in fields:
+                obj_migrated.pop(field, None)
+
+    if to_version_tuple < version2tuple("0.9.0") <= obj_version_tuple:
+        # StandardsType.version and .extensions were added in 0.9.0.
+        # assetsSummary is computed by the server, so it is safe to strip them.
+        assets_summary = obj_migrated.get("assetsSummary") or {}
+        if standards := assets_summary.get("dataStandard"):
+            stripped: List[dict] = []
+            for standard in standards:
+                standard = {
+                    k: v
+                    for k, v in standard.items()
+                    if k not in ("version", "extensions")
+                }
+                if standard not in stripped:
+                    stripped.append(standard)
+            assets_summary["dataStandard"] = stripped
+
     # Always update schemaVersion when migrating
     obj_migrated["schemaVersion"] = to_version
     return obj_migrated
@@ -526,15 +562,59 @@ def add_if_missing(standard: dict) -> None:
         if standard not in stats["dataStandard"]:
             stats["dataStandard"].append(standard)
 
+    # Collect per-asset dataStandard declarations (populated by dandi-cli)
+    for standard in assetmeta.get("dataStandard") or []:
+        add_if_missing(standard)
+
+    # Heuristic fallbacks for assets uploaded by older clients which did not
+    # populate per-asset dataStandard.  If an asset does declare a standard,
+    # possibly with a version, the bare entry added here is pruned by
+    # _merge_data_standards().
+    # TODO: remove once all assets on the server have dataStandard populated.
     if "nwb" in assetmeta["encodingFormat"]:
         add_if_missing(models.nwb_standard)
-    # TODO: RF assumption that any .json implies BIDS
     if Path(assetmeta["path"]).name == "dataset_description.json":
         add_if_missing(models.bids_standard)
     if Path(assetmeta["path"]).suffixes == [".ome", ".zarr"]:
         add_if_missing(models.ome_ngff_standard)
 
 
+def _merge_data_standards(standards: List[dict]) -> List[dict]:
+    """Merge per-asset dataStandard entries into a list for AssetsSummary
+
+    Produces a single entry per standard (as identified by its ``identifier``,
+    or ``name`` if there is none) and version, with ``extensions`` united
+    across all assets.  An entry without a version and extensions is dropped
+    if the same standard is also listed with a version, so that assets from
+    older clients (see the heuristics in ``_add_asset_to_stats``) do not
+    produce duplicates.
+    """
+    merged: Dict[tuple, dict] = {}
+    for standard in standards:
+        key = (
+            standard.get("identifier") or standard.get("name"),
+            standard.get("version"),
+        )
+        entry = merged.setdefault(
+            key, {k: v for k, v in standard.items() if k != "extensions"}
+        )
+        for ext in standard.get("extensions") or []:
+            exts = entry.setdefault("extensions", [])
+            if ext not in exts:
+                exts.append(ext)
+    versioned = {std_id for std_id, version in merged if version is not None}
+    result = []
+    for (std_id, version), entry in merged.items():
+        if "extensions" in entry:
+            entry["extensions"].sort(
+                key=lambda e: (e.get("name") or "", e.get("version") or "")
+            )
+        elif version is None and std_id in versioned:
+            continue
+        result.append(entry)
+    return result
+
+
 # TODO?: move/bind such helpers as .from_metadata or alike within
 #        model classes themselves to centralize access to those constructors.
 def aggregate_assets_summary(metadata: Iterable[Dict[str, Any]]) -> dict:
@@ -553,4 +633,6 @@ def aggregate_assets_summary(metadata: Iterable[Dict[str, Any]]) -> dict:
         len(stats.pop("tissuesample", [])) + len(stats.pop("slice", []))
     ) or None
     stats["numberOfCells"] = len(stats.pop("cell", [])) or None
+    if "dataStandard" in stats:
+        stats["dataStandard"] = _merge_data_standards(stats["dataStandard"])
     return models.AssetsSummary(**stats).model_dump(mode="json", exclude_none=True)

dandischema/models.py

@@ -859,11 +859,28 @@ class MeasurementTechniqueType(BaseType):
 class StandardsType(BaseType):
     """Identifier for data standard used"""
 
+    version: Optional[str] = Field(
+        None,
+        description="Version of the standard used.",
+        json_schema_extra={"nskey": "schema"},
+    )
+    extensions: Optional[List[StandardsType]] = Field(
+        None,
+        description="Extensions to the standard used "
+        "(e.g. NWB extensions like ndx-*, HED library schemas).",
+        json_schema_extra={"nskey": DANDI_NSKEY},
+    )
+    # TODO: consider how to formalize BIDS extensions (BEPs) once BIDS
+    # has a machine-readable way to declare them.
     schemaKey: Literal["StandardsType"] = Field(
         "StandardsType", validate_default=True, json_schema_extra={"readOnly": True}
     )
 
 
+# Self-referencing model needs rebuild after class definition
+# https://docs.pydantic.dev/latest/concepts/postponed_annotations/#self-referencing-or-recursive-models
+StandardsType.model_rebuild()
+
 nwb_standard = StandardsType(
     name="Neurodata Without Borders (NWB)",
     identifier="RRID:SCR_015242",
@@ -879,6 +896,11 @@ class StandardsType(BaseType):
     identifier="DOI:10.25504/FAIRsharing.9af712",
 ).model_dump(mode="json", exclude_none=True)
 
+hed_standard = StandardsType(
+    name="Hierarchical Event Descriptors (HED)",
+    identifier="RRID:SCR_014074",
+).model_dump(mode="json", exclude_none=True)
+
 
 class ContactPoint(DandiBaseModel):
     email: Optional[EmailStr] = Field(
@@ -1934,6 +1956,12 @@ class BareAsset(CommonModel):
         json_schema_extra={"nskey": "prov"},
     )
 
+    dataStandard: Optional[List[StandardsType]] = Field(
+        None,
+        description="Data standard(s) applicable to this asset.",
+        json_schema_extra={"nskey": DANDI_NSKEY},
+    )
+
     schemaKey: Literal["BareAsset"] = Field(
         "BareAsset", validate_default=True, json_schema_extra={"readOnly": True}
     )