diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index 496a1bcd..1ce53bed 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -21,10 +21,10 @@ jobs: name: py${{ matrix.python-version }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} #---------------------------------------------- @@ -44,7 +44,7 @@ jobs: #---------------------------------------------- - name: Load cached venv id: cached-poetry-dependencies - uses: actions/cache@v4 + uses: actions/cache@v6 with: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 876ba15d..056eaeb6 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -24,10 +24,10 @@ jobs: name: ${{ matrix.os }} py${{ matrix.python-version }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} @@ -46,7 +46,7 @@ jobs: #---------------------------------------------- - name: Load cached venv id: cached-pip-wheels - uses: actions/cache@v4 + uses: actions/cache@v6 with: path: ~/.cache key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml index 1cc33d54..e5a96978 100644 --- a/.github/workflows/ruff.yml +++ b/.github/workflows/ruff.yml @@ -4,5 +4,5 @@ jobs: ruff: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 - - uses: astral-sh/ruff-action@v3 + - uses: actions/checkout@v7 + - uses: astral-sh/ruff-action@v4.1.0 diff --git a/docs/api.rst b/docs/api.rst index 185bec7c..966bc310 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -14,6 +14,7 @@ API documentation api/codelist api/regionprocessor api/datavalidator + api/metavalidator api/requireddatavalidator api/countries api/nuts diff --git a/docs/api/metavalidator.rst b/docs/api/metavalidator.rst new file mode 100644 index 00000000..eec2a3a4 --- /dev/null +++ b/docs/api/metavalidator.rst @@ -0,0 +1,56 @@ +.. _meta-validation: + +.. currentmodule:: nomenclature + +**MetaValidator** +================= + +**Meta validation** checks if meta indicators follow allowed values and ranges. + +Consider the example below: + +.. code:: yaml + + - name: Sustainability Concern|Exceeding Prudent Limit For Geological Carbon Storage|World + meta: Emissions Diagnostics|Cumulative CCS [2020-2100, Gt CO2] + validation: + - warning_level: high + upper_bound: 1490 + - warning_level: medium + upper_bound: 1290 + - meta: Project + values: [Project Name 1, Project Name 2] + + +Each criteria item contains **meta indicator filter arguments** and **validation arguments**. + +The meta indicator filter argument ``meta`` (also allowed as ``meta_columns_to_validate``) +refers to the column(s) in the meta indicator table that will undergo validation. +For the first criteria item, validation will check the values of column +*"Emissions Diagnostics|Cumulative CCS [2020-2100, Gt CO2]"*. +For the second criteria item, validation will check column *"Project"*. +If multiple columns are specified, the validation fails if *any* value for a given +row fails (e.g.: if Column A fails validation and Column B doesn't, the row +is flagged as failed). +The ``name`` field specifies the meta indicator column that will be added post-validation +with the validation results (``ok``, ``low``, ``medium``, ``high``, ``error``). + +The ``validation`` arguments follow the same rules as :class:`DataValidator` +(see :ref:`data-validation`), but apply exclusively to meta indicator columns. +In addition, the ``values`` field supports membership checks. In the example +above, the "Project" meta indicator will be checked for its values being either +*"Project Name 1"* or *"Project Name 2"*. + +Standard usage +-------------- + +.. code-block:: python + + from nomenclature import MetaValidator + + # ...setting directory/file paths and loading dataset + + MetaValidator.from_file(yaml_file_containing_meta_validation_criteria).apply(df) + +.. autoclass:: MetaValidator + :members: from_file, from_codelist, apply, validate_with_definition diff --git a/nomenclature/cli.py b/nomenclature/cli.py index 258c44c6..cb622a3a 100644 --- a/nomenclature/cli.py +++ b/nomenclature/cli.py @@ -76,6 +76,9 @@ def validate_project( validate_data: Annotated[ str | None, typer.Option(help="Name of data validation folder") ] = None, + validate_meta: Annotated[ + str | None, typer.Option(help="Name of meta validation folder") + ] = None, dimensions: Annotated[ list[str] | None, typer.Option("--dimension", help="Dimensions to check (defaults to all)"), @@ -88,7 +91,7 @@ def validate_project( - YAML syntax validation for all files - Parsing of codelists in the definitions folder - Validation of model mappings against region codelists - - Consistency checks for required-data and data-validation criteria + - Consistency checks for required-data, data-validation, and meta criteria \b Example: @@ -96,7 +99,13 @@ def validate_project( """ assert_valid_yaml(path) assert_valid_structure( - path, definitions, mappings, required_data, validate_data, dimensions + path, + definitions, + mappings, + required_data, + validate_data, + validate_meta, + dimensions, ) diff --git a/nomenclature/code.py b/nomenclature/code.py index d7aa5530..e8e8c234 100644 --- a/nomenclature/code.py +++ b/nomenclature/code.py @@ -18,7 +18,7 @@ from nomenclature.countries import countries -# This must be kept in sync with the subtypes of `DataValidationCriteria` +# This must be kept in sync with the subtypes of `ValidationCriteria` VALIDATION_ARGS = [ "upper_bound", "lower_bound", @@ -431,4 +431,18 @@ class MetaCode(Code): """ - allowed_values: list[Any] | None = None + value: list[Any] | None = Field(default=None, alias="allowed_values") + model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) + + @property + def has_validation_args(self) -> bool: + return True if self.validation_args else False + + @property + def validation_args(self) -> dict: + alias_map = {"allowed_values": "value"} # Map alias to canonical name + return { + alias_map.get(key, key): value + for key, value in self.extra_attributes.items() + if alias_map.get(key, key) in VALIDATION_ARGS and value is not None + } diff --git a/nomenclature/codelist.py b/nomenclature/codelist.py index 23a4ca32..25d30eaa 100644 --- a/nomenclature/codelist.py +++ b/nomenclature/codelist.py @@ -1030,7 +1030,7 @@ def _parse_region_code_dir( class MetaCodeList(CodeList): - """A subclass of CodeList specified for MetaCodes + """A subclass of CodeList specified for meta indicators Attributes ---------- diff --git a/nomenclature/config.py b/nomenclature/config.py index 0204a7c7..f3dd2899 100644 --- a/nomenclature/config.py +++ b/nomenclature/config.py @@ -328,9 +328,11 @@ def check_datetime_format(self, df: IamDataFrame) -> None: "The following datetime values are invalid:", errors ) - def validate_datetime(self, df: IamDataFrame, dimensions: list[str] | None = None) -> None: + def validate_datetime( + self, df: IamDataFrame, dimensions: list[str] | None = None + ) -> None: """Validate datetime coordinates against allowed format and/or timezone. - + Parameters ---------- df : IamDataFrame @@ -338,11 +340,13 @@ def validate_datetime(self, df: IamDataFrame, dimensions: list[str] | None = Non dimensions : list of str, optional List of allowed dimensions for validation """ - if "subannual" in df.data.columns and (dimensions is None or "subannual" not in dimensions): + if "subannual" in df.data.columns and ( + dimensions is None or "subannual" not in dimensions + ): raise TimeDomainError( "Invalid time domain - `subannual` found, but not allowed." ) - + if df.time_domain == "year": if not self.year_allowed: raise TimeDomainError( diff --git a/nomenclature/exceptions.py b/nomenclature/exceptions.py index 87a8a86a..28aa8ca6 100644 --- a/nomenclature/exceptions.py +++ b/nomenclature/exceptions.py @@ -174,6 +174,20 @@ def __init__( super().__init__(message) +class MetaValidationError(NoTracebackException): + def __init__( + self, + fail_list: list[str], + file: Path, + ) -> None: + failed_criteria_info = "\n".join(fail_list) + message = ( + f"Meta indicator validation failed with error(s) (file: {get_relative_path(file)}):\n" + f"{failed_criteria_info}" + ) + super().__init__(message) + + class RequiredDataMissingError(ValueError): def __init__(self, missing_data_info: str, file: Path) -> None: message = ( diff --git a/nomenclature/processor/__init__.py b/nomenclature/processor/__init__.py index 291b7c53..4d096bd9 100644 --- a/nomenclature/processor/__init__.py +++ b/nomenclature/processor/__init__.py @@ -1,9 +1,11 @@ from nomenclature.processor.processor import Processor # noqa +from nomenclature.processor.validator import Validator # noqa from nomenclature.processor.region import ( # noqa RegionAggregationMapping, RegionProcessor, ) from nomenclature.processor.nuts import NutsProcessor # noqa from nomenclature.processor.required_data import RequiredDataValidator # noqa -from nomenclature.processor.data_validator import DataValidator # noqa +from nomenclature.processor.data import DataValidator # noqa +from nomenclature.processor.meta import MetaValidator # noqa from nomenclature.processor.aggregator import Aggregator # noqa diff --git a/nomenclature/processor/data.py b/nomenclature/processor/data.py new file mode 100644 index 00000000..c4f7c8f5 --- /dev/null +++ b/nomenclature/processor/data.py @@ -0,0 +1,194 @@ +import logging +import textwrap +from enum import IntEnum +from pathlib import Path + +import pandas as pd +from toolkit.exceptions import NoTracebackException +import yaml +from pyam import IamDataFrame + +from nomenclature.codelist import VariableCodeList +from nomenclature.definition import DataStructureDefinition +from nomenclature.exceptions import DataValidationError, NoTracebackExceptionGroup +from nomenclature.processor.validator import ( + ValidationValue, + ValidationRange, + ValidationBounds, + ValidationItem, +) +from nomenclature.processor import Validator +from nomenclature.processor.iamc import IamcDataFilter +from nomenclature.utils import get_relative_path + +logger = logging.getLogger(__name__) + + +class WarningEnum(IntEnum): + error = 50 + high = 40 + medium = 30 + low = 20 + + +class DataValidationItem(ValidationItem, IamcDataFilter): + validation: list[ValidationValue | ValidationBounds | ValidationRange] + + def apply( + self, df: IamDataFrame, fail_list: list, output_list: list + ) -> tuple[bool, list, list]: + """Apply data validation to IamDataFrame.""" + error = False + per_item_df = df.filter(**self.filter_args) + + # If name is given, set a meta indicator for the item being processed + if self.name is not None: + meta_index = per_item_df.index.copy() + df.set_meta(name=self.name, meta="ok", index=meta_index) + + for criterion in self.validation: + failed_validation = per_item_df.validate(**criterion.validation_args) + if failed_validation is not None: + per_item_df = IamDataFrame( + pd.concat([per_item_df.data, failed_validation]).drop_duplicates( + keep=False + ) + ) + + # Mark failing scenarios with a meta indicator and warning level + failed_index = failed_validation.set_index( + ["model", "scenario"] + ).index.drop_duplicates() + + if self.name is not None: + df.set_meta( + name=self.name, + meta=criterion.warning_level.name, + index=meta_index.intersection(failed_index), + ) + # Remove failed scenarios from the meta index to avoid + # lower warnings overriding higher warnings in meta indicators + meta_index = meta_index.difference(failed_index) + + failed_validation["warning_level"] = criterion.warning_level.name + failed_validation["criteria"] = str(criterion) + output_list.append(failed_validation) + if criterion.warning_level == WarningEnum.error: + error = True + fail_list.append(" Criteria: " + str(self) + ", " + str(criterion)) + fail_list.append( + textwrap.indent( + failed_validation.iloc[:, :-1].to_string(), prefix=" " + ) + + "\n" + ) + return error, fail_list, output_list + + +class DataValidator(Validator): + """Processor for validating IAMC datapoints""" + + criteria_items: list[DataValidationItem] + file: Path | str + output_path: Path | None = None + exception_cls: type[NoTracebackException] = DataValidationError + + @classmethod + def from_file( + cls, file: Path | str, output_path: Path | str | None = None + ) -> "DataValidator": + """Create a :class:`DataValidator` from a YAML file. + + Parameters + ---------- + file : :class:`pathlib.Path` or str + Path to the YAML file containing the validation criteria. + output_path : :class:`pathlib.Path` or str, optional + Path to write an Excel file with all flagged datapoints. + + Returns + ------- + DataValidator + """ + with open(file, "r", encoding="utf-8") as f: + content = yaml.safe_load(f) + criteria_items = [] + for item in content: + # Simple case where filter and criteria args are all given at top level + if "validation" not in item: + item["validation"] = [dict()] + + # If some criteria args are given at top-level, add to "validation" list + criteria = [ + criterion + for criterion in item + if criterion + not in list(IamcDataFilter.model_fields) + ["name", "validation"] + ] + for criterion in criteria: + value = item.pop(criterion) + for criteria_item in item["validation"]: + criteria_item[criterion] = value + criteria_items.append(item) + + return cls(file=file, criteria_items=criteria_items, output_path=output_path) # type: ignore + + @classmethod + def from_codelist( + cls, codelist: VariableCodeList, output_path: Path | None = None + ) -> "DataValidator": + """Create a :class:`DataValidator` from a :class:`~nomenclature.codelist.VariableCodeList`. + + Extracts validation criteria from variables in the codelist that define + bounds or tolerance ranges. + + Parameters + ---------- + codelist : VariableCodeList + Variable codelist containing validation arguments. + output_path : :class:`pathlib.Path`, optional + Path to write an Excel file with all flagged datapoints. + + Returns + ------- + DataValidator + """ + criteria_items = [ + { + "variable": variable.name, + "validation": [variable.validation_args], + } + for variable in codelist.values() + if variable.has_validation_args + ] + return cls( + file="definitions", criteria_items=criteria_items, output_path=output_path + ) + + def validate_with_definition(self, dsd: DataStructureDefinition) -> None: + """Validate the criteria items against a :class:`DataStructureDefinition`. + + Checks that all variables and regions referenced in the criteria + exist in the provided definition. + + Parameters + ---------- + dsd : DataStructureDefinition + Definition against which to check the items of the Validator + + Raises + ------ + ExceptionGroup + If any criteria item references unknown variables or regions. + """ + errors: list[Exception] = [] + for criterion in self.criteria_items: + try: + criterion.validate_with_definition(dsd) + except NoTracebackExceptionGroup as exception: + errors.extend(exception.exceptions) + if errors: + raise NoTracebackExceptionGroup( + f"Error in DataValidator (file {get_relative_path(self.file)})", + errors, + ) diff --git a/nomenclature/processor/data_validator.py b/nomenclature/processor/data_validator.py deleted file mode 100644 index c91fe846..00000000 --- a/nomenclature/processor/data_validator.py +++ /dev/null @@ -1,390 +0,0 @@ -import logging -import textwrap -from enum import IntEnum -from pathlib import Path - -import pandas as pd -import yaml -from pyam import IamDataFrame -from pyam.utils import adjust_log_level -from pydantic import ( - BaseModel, - ConfigDict, - Field, - computed_field, - field_validator, - model_validator, -) - -from nomenclature.codelist import VariableCodeList -from nomenclature.definition import DataStructureDefinition -from nomenclature.exceptions import DataValidationError, NoTracebackExceptionGroup -from nomenclature.processor import Processor -from nomenclature.processor.iamc import IamcDataFilter -from nomenclature.utils import get_relative_path - -logger = logging.getLogger(__name__) - - -class WarningEnum(IntEnum): - error = 50 - high = 40 - medium = 30 - low = 20 - - -class DataValidationCriteria(BaseModel): - warning_level: WarningEnum = WarningEnum.error - - model_config = ConfigDict(extra="forbid") - - @field_validator("warning_level", mode="before") - @classmethod - def validate_warning_level(cls, value): - if isinstance(value, str): - try: - return WarningEnum[value] - except KeyError: - raise ValueError( - f"Invalid warning level: {value}. Expected one of:" - f" {', '.join(level.name for level in WarningEnum)}" - ) - return value - - @property - def criteria(self): - pass - - def __str__(self): - return ", ".join([f"{key}: {value}" for key, value in self.criteria.items()]) - - -class DataValidationValue(DataValidationCriteria): - value: float - rtol: float = 0.0 - atol: float = 0.0 - - @property - def tolerance(self) -> float: - return self.value * self.rtol + self.atol - - @computed_field - def upper_bound(self) -> float: - return self.value + self.tolerance - - @computed_field - def lower_bound(self) -> float: - return self.value - self.tolerance - - @property - def validation_args(self): - """Attributes used for validation (as bounds).""" - return self.model_dump( - exclude_none=True, - exclude_unset=True, - exclude=["warning_level", "value", "rtol", "atol"], - ) - - @property - def criteria(self): - """Attributes used for validation (as specified in the file).""" - return self.model_dump( - exclude_none=True, - exclude_unset=True, - exclude=["warning_level", "lower_bound", "upper_bound"], - ) - - -class DataValidationBounds(DataValidationCriteria): - upper_bound: float | None = None - lower_bound: float | None = None - - # Allow extra but raise error to guard against multiple criteria - model_config = ConfigDict(extra="allow") - - @model_validator(mode="after") - def check_validation_criteria_exist(self): - if self.upper_bound is None and self.lower_bound is None: - raise ValueError("No validation criteria provided: " + str(self.criteria)) - return self - - @model_validator(mode="after") - def check_validation_multiple_criteria(self): - if self.model_extra: - raise ValueError( - "Must use either bounds, range or value, found: " + str(self.criteria) - ) - return self - - @property - def validation_args(self): - return self.criteria - - @property - def criteria(self): - return self.model_dump( - exclude_none=True, exclude_unset=True, exclude=["warning_level"] - ) - - -class DataValidationRange(DataValidationCriteria): - range: list[float] = Field(..., min_length=2, max_length=2) - - @field_validator("range", mode="after") - @classmethod - def check_range_is_valid(cls, value: list[float]): - if value[0] > value[1]: - raise ValueError( - "Validation 'range' must be given as `(lower_bound, upper_bound)`, " - "found: " + str(value) - ) - return value - - @computed_field - def upper_bound(self) -> float: - return self.range[1] - - @computed_field - def lower_bound(self) -> float: - return self.range[0] - - @property - def validation_args(self): - """Attributes used for validation (as bounds).""" - return self.model_dump( - exclude_none=True, - exclude_unset=True, - exclude=["warning_level", "range"], - ) - - @property - def criteria(self): - return self.model_dump( - exclude_none=True, - exclude_unset=True, - exclude=["warning_level", "lower_bound", "upper_bound"], - ) - - -class DataValidationItem(IamcDataFilter): - name: str | None = None - validation: list[DataValidationValue | DataValidationRange | DataValidationBounds] - - @model_validator(mode="after") - def check_warnings_order(self): - """Check if warnings are set in descending order of severity.""" - if self.validation != sorted( - self.validation, key=lambda c: c.warning_level, reverse=True - ): - raise ValueError( - f"Validation criteria for {self.criteria} not sorted" - " in descending order of severity." - ) - else: - return self - - @property - def filter_args(self): - """Attributes used for validation (as specified in the file).""" - return self.model_dump( - exclude_none=True, exclude_unset=True, exclude=["validation", "name"] - ) - - def __str__(self): - return ", ".join([f"{key}: {value}" for key, value in self.filter_args.items()]) - - def apply( - self, df: IamDataFrame, fail_list: list, output_list: list - ) -> tuple[bool, list, list]: - error = False - per_item_df = df.filter(**self.filter_args) - - # If name is given, set a meta indicator for the item being processed - if self.name is not None: - meta_index = per_item_df.index.copy() - df.set_meta(name=self.name, meta="ok", index=meta_index) - - for criterion in self.validation: - failed_validation = per_item_df.validate(**criterion.validation_args) - if failed_validation is not None: - per_item_df = IamDataFrame( - pd.concat([per_item_df.data, failed_validation]).drop_duplicates( - keep=False - ) - ) - - # Mark failing scenarios with a meta indicator and warning level - failed_index = failed_validation.set_index( - ["model", "scenario"] - ).index.drop_duplicates() - - if self.name is not None: - df.set_meta( - name=self.name, - meta=criterion.warning_level.name, - index=meta_index.intersection(failed_index), - ) - # Remove failed scenarios from the meta index to avoid - # lower warnings overriding higher warnings in meta indicators - meta_index = meta_index.difference(failed_index) - - failed_validation["warning_level"] = criterion.warning_level.name - failed_validation["criteria"] = str(criterion) - output_list.append(failed_validation) - if criterion.warning_level == WarningEnum.error: - error = True - fail_list.append(" Criteria: " + str(self) + ", " + str(criterion)) - fail_list.append( - textwrap.indent( - failed_validation.iloc[:, :-1].to_string(), prefix=" " - ) - + "\n" - ) - return error, fail_list, output_list - - -class DataValidator(Processor): - """Processor for validating IAMC datapoints""" - - criteria_items: list[DataValidationItem] - file: Path | str - output_path: Path | None = None - - @classmethod - def from_file( - cls, file: Path | str, output_path: Path | str | None = None - ) -> "DataValidator": - """Create a :class:`DataValidator` from a YAML file. - - Parameters - ---------- - file : :class:`pathlib.Path` or str - Path to the YAML file containing the validation criteria. - output_path : :class:`pathlib.Path` or str, optional - Path to write an Excel file with all flagged datapoints. - - Returns - ------- - DataValidator - """ - with open(file, "r", encoding="utf-8") as f: - content = yaml.safe_load(f) - criteria_items = [] - for item in content: - # Simple case where filter and criteria args are all given at top level - if "validation" not in item: - item["validation"] = [dict()] - - # If some criteria args are given at top-level, add to "validation" list - criteria = [ - criterion - for criterion in item - if criterion - not in list(IamcDataFilter.model_fields) + ["validation", "name"] - ] - for criterion in criteria: - value = item.pop(criterion) - for criteria_item in item["validation"]: - criteria_item[criterion] = value - criteria_items.append(item) - - return cls(file=file, criteria_items=criteria_items, output_path=output_path) # type: ignore - - @classmethod - def from_codelist( - cls, codelist: VariableCodeList, output_path: Path | None = None - ) -> "DataValidator": - """Create a :class:`DataValidator` from a :class:`~nomenclature.codelist.VariableCodeList`. - - Extracts validation criteria from variables in the codelist that define - bounds or tolerance ranges. - - Parameters - ---------- - codelist : VariableCodeList - Variable codelist containing validation arguments. - output_path : :class:`pathlib.Path`, optional - Path to write an Excel file with all flagged datapoints. - - Returns - ------- - DataValidator - """ - criteria_items = [ - { - "variable": variable.name, - "validation": [variable.validation_args], - } - for variable in codelist.values() - if variable.has_validation_args - ] - return cls( - file="definitions", criteria_items=criteria_items, output_path=output_path - ) - - def apply(self, df: IamDataFrame) -> IamDataFrame: - """Validates data in IAMC format according to specified criteria. - - Logs warning/error messages for each criterion that is not met. - - Parameters - ---------- - df : pyam.IamDataFrame - Data in IAMC format to be validated - - Returns - ------- - pyam.IamDataFrame - - Raises - ------ - :exc:`ValueError` if any criterion has a warning level of ``error`` - """ - - error_list: list[bool] = [] - fail_list: list[str] = [] - output_list: list[pd.DataFrame] = [] - - with adjust_log_level(): - for item in self.criteria_items: - error, fail_list, output_list = item.apply(df, fail_list, output_list) - error_list.append(error) - if self.output_path: - pd.concat(output_list).to_excel(self.output_path, index=False) - fail_msg = "(file %s):\n" % get_relative_path(self.file) - if any(error_list): - raise DataValidationError(fail_list, self.file) - if fail_list: - fail_msg = ( - "Data validation with warning(s) " + fail_msg + "\n".join(fail_list) - ) - logger.warning(fail_msg) - return df - - def validate_with_definition(self, dsd: DataStructureDefinition) -> None: - """Validate the criteria items against a :class:`DataStructureDefinition`. - - Checks that all variables and regions referenced in the criteria - exist in the provided definition. - - Parameters - ---------- - dsd : DataStructureDefinition - Data structure definition to validate against. - - Raises - ------ - ExceptionGroup - If any criteria item references unknown variables or regions. - """ - errors: list[Exception] = [] - for criterion in self.criteria_items: - try: - criterion.validate_with_definition(dsd) - except NoTracebackExceptionGroup as exception: - errors.extend(exception.exceptions) - if errors: - raise NoTracebackExceptionGroup( - f"Error in DataValidator (file {get_relative_path(self.file)})", - errors, - ) diff --git a/nomenclature/processor/iamc.py b/nomenclature/processor/iamc.py index bf1b1c0f..72b127af 100644 --- a/nomenclature/processor/iamc.py +++ b/nomenclature/processor/iamc.py @@ -2,6 +2,7 @@ from pydantic import BaseModel, ConfigDict, field_validator from toolkit.exceptions import NoTracebackException +from nomenclature.codelist import CodeList from nomenclature.definition import DataStructureDefinition from nomenclature.exceptions import NoTracebackExceptionGroup @@ -26,11 +27,11 @@ def criteria(self): return self.model_dump(exclude_none=True, exclude_unset=True) def validate_with_definition(self, dsd: DataStructureDefinition) -> None: - errors = [] - + """Check dimensions to validate against the DataStructureDefinition""" + errors: list[NoTracebackException] = [] # Check for filter-items that are not defined in the codelists for dimension in IAMC_IDX: - codelist = getattr(dsd, dimension, None) + codelist: CodeList | None = getattr(dsd, dimension, None) # No validation if codelist is not defined or filter-item is None if codelist is None or getattr(self, dimension) is None: continue diff --git a/nomenclature/processor/meta.py b/nomenclature/processor/meta.py index 85d8df03..92e2166f 100644 --- a/nomenclature/processor/meta.py +++ b/nomenclature/processor/meta.py @@ -1,20 +1,140 @@ -from pathlib import Path +import logging +import textwrap +from typing import Any +import pandas as pd import pyam -from nomenclature.processor import Processor +import yaml + +from pathlib import Path + +from pydantic import BaseModel, ConfigDict, Field, field_validator +from pyam import IamDataFrame +from pyam.utils import adjust_log_level +from nomenclature.definition import DataStructureDefinition from nomenclature.codelist import MetaCodeList +from nomenclature.exceptions import MetaValidationError +from nomenclature.processor import Validator +from nomenclature.processor.validator import ( + ValidationBounds, + ValidationRange, + ValidationValue, + ValidationItem, + WarningEnum, +) +from nomenclature.utils import get_relative_path +from toolkit.exceptions import NoTracebackException, NoTracebackExceptionGroup +logger = logging.getLogger(__name__) -class MetaValidator(Processor): - """Meta indicator validation and processing class""" - meta_code_list: MetaCodeList +class MetaFilter(BaseModel): + meta: list[str] = Field(..., alias="meta_columns_to_validate") + + model_config = ConfigDict( + validate_by_alias=True, validate_by_name=True, extra="forbid" + ) + + @field_validator("meta", mode="before") + @classmethod + def single_input_to_list(cls, v): + return v if isinstance(v, list) else [v] + + @property + def criteria(self): + return self.model_dump(exclude_none=True, exclude_unset=True) - def __init__(self, path_to_meta_code_list_files: Path): - super().__init__( - meta_code_list=MetaCodeList.from_directory( - name="meta_code_list", path=path_to_meta_code_list_files + def validate_with_definition(self, dsd: DataStructureDefinition) -> None: + """Check criteria items against the DataStructureDefinition""" + codelist: MetaCodeList | None = getattr(dsd, "meta", None) + # No validation if codelist is not defined or filter-item is None + errors: list[NoTracebackException] = [] + if codelist is None: + return + if invalid := codelist.validate_items(getattr(self, "meta")): + errors.append( + NoTracebackException( + "The following meta indicators are not defined in the " + "DataStructureDefinition:\n " + + ", ".join(f"'{item}'" for item in invalid) + ) ) - ) + raise NoTracebackExceptionGroup( + f"Errors in {self.__class__.__name__}", errors + ) + + +class MetaValidationValue(ValidationValue): + value: float | list[Any] = Field(..., alias="values") + + model_config = ConfigDict( + validate_by_alias=True, validate_by_name=True, extra="forbid" + ) + + @field_validator("value", mode="after") + @classmethod + def coerce_str_to_list_str(cls, v): + if isinstance(v, (float, list)): + return v + if isinstance(v, str): + return [v] + + +class MetaValidationItem(ValidationItem, MetaFilter): + """Validation item for meta indicator validation""" + + validation: list[MetaValidationValue | ValidationBounds | ValidationRange] + + def apply(self, df: IamDataFrame, fail_list: list, output_list: list): + """Apply meta validation to IamDataFrame.""" + error = False + per_item_df = df.meta.filter(self.meta, axis="columns") + + # If name is given, set a meta indicator for the item being processed + if self.name is not None: + meta_index: pd.MultiIndex = per_item_df.index + df.set_meta(name=self.name, meta="ok", index=meta_index) + + for criterion in self.validation: + failed_validation = _validate_meta(per_item_df, **criterion.validation_args) + if failed_validation is not None: + # Create a new meta DataFrame with failed validation rows removed + per_item_df = per_item_df.loc[ + ~per_item_df.index.isin(failed_validation.index) + ] + + # Mark failing scenarios with a meta indicator and warning level + failed_index: pd.MultiIndex = failed_validation.index.drop_duplicates() + + if self.name is not None: + df.meta.loc[failed_index.values, self.name] = ( + criterion.warning_level.name + ) + # Remove failed scenarios from the meta index to avoid + # lower warnings overriding higher warnings in meta indicators + meta_index = meta_index.difference(failed_index) + + failed_validation["warning_level"] = criterion.warning_level.name + failed_validation["criteria"] = str(criterion) + output_list.append(failed_validation) + if criterion.warning_level == WarningEnum.error: + error = True + fail_list.append(" Criteria: " + str(self) + ", " + str(criterion)) + fail_list.append( + textwrap.indent( + failed_validation.iloc[:, :-1].to_string(), prefix=" " + ) + + "\n" + ) + return error, fail_list, output_list + + +class MetaValidator(Validator): + """Meta indicator validation and processing class""" + + criteria_items: list[MetaValidationItem] + file: Path | str + output_path: Path | None = None + exception_cls: type[NoTracebackException] = MetaValidationError def _values_allowed(self, values, allowed_values, meta_indicator) -> bool: """Checks if the values within a meta indicator column are @@ -50,6 +170,73 @@ def _values_allowed(self, values, allowed_values, meta_indicator) -> bool: ) return True + @classmethod + def from_file( + cls, file: Path | str, output_path: Path | str | None = None + ) -> "MetaValidator": + """Create a :class:`MetaValidator` from a YAML file. + + Parameters + ---------- + file : :class:`pathlib.Path` or str + Path to the YAML file containing the validation criteria. + output_path : :class:`pathlib.Path` or str, optional + Path to write an Excel file with all flagged datapoints. + + Returns + ------- + MetaValidator + """ + with open(file, "r", encoding="utf-8") as f: + content = yaml.safe_load(f) + criteria_items = [] + for item in content: + # Simple case where filter and criteria args are all given at top level + if "validation" not in item: + item["validation"] = [dict()] + + # If some criteria args are given at top-level, add to "validation" list + criteria = [ + criterion + for criterion in item + if criterion not in ["name", "meta", "validation"] + ] + for criterion in criteria: + value = item.pop(criterion) + for criteria_item in item["validation"]: + criteria_item[criterion] = value + criteria_items.append(item) + + return cls(file=file, criteria_items=criteria_items, output_path=output_path) # type: ignore + + @classmethod + def from_codelist( + cls, codelist: MetaCodeList, output_path: Path | None = None + ) -> "MetaValidator": + """Create a MetaValidator from a MetaCodeList + + Parameters + ---------- + codelist : MetaCodeList + The MetaCodeList to use for validation + + Returns + ------- + MetaValidator + A new MetaValidator instance with the given MetaCodeList + """ + criteria_items = [ + { + "meta": [meta.name], + "validation": [meta.validation_args], + } + for meta in codelist.values() + if meta.has_validation_args + ] + return cls( + criteria_items=criteria_items, file="definitions", output_path=output_path + ) + def apply(self, df: pyam.IamDataFrame) -> pyam.IamDataFrame: """Apply meta indicator validation processing @@ -71,24 +258,88 @@ def apply(self, df: pyam.IamDataFrame) -> pyam.IamDataFrame: definition file """ - if invalid_meta_indicators := [ - meta_indicator - for meta_indicator in df.meta.columns - if meta_indicator not in self.meta_code_list.mapping - ]: - raise ValueError( - f"Invalid meta indicator: {repr_list(invalid_meta_indicators)}\n" - f"Valid meta indicators: {repr_list(self.meta_code_list.mapping.keys())}" - ) + error_list: list[bool] = [] + fail_list: list[str] = [] + output_list: list[pd.DataFrame] = [] - for meta_indicator in df.meta.columns: - self._values_allowed( - list(set(df.meta[meta_indicator].values)), - self.meta_code_list.mapping[meta_indicator].allowed_values, - meta_indicator, - ) + with adjust_log_level(): + for item in self.criteria_items: + error, fail_list, output_list = item.apply(df, fail_list, output_list) + error_list.append(error) + if self.output_path: + pd.concat(output_list).to_excel(self.output_path, index=False) + fail_msg = f"(file {get_relative_path(self.file)}):\n" + if any(error_list): + raise MetaValidationError(fail_list, self.file) + if fail_list: + fail_msg = ( + "Meta validation with warning(s) " + fail_msg + "\n".join(fail_list) + ) + logger.warning(fail_msg) return df + def validate_with_definition(self, dsd: DataStructureDefinition) -> None: + errors: list[Exception] = [] + for criterion in self.criteria_items: + try: + criterion.validate_with_definition(dsd) + except NoTracebackExceptionGroup as exception: + errors.extend(exception.exceptions) + if errors: + raise NoTracebackExceptionGroup( + f"Error in MetaValidator (file {get_relative_path(self.file)})", + errors, + ) + def repr_list(x): return "'" + "', '".join(map(str, x)) + "'" + + +def _validate_meta(df: pd.DataFrame, **kwargs) -> pd.DataFrame | None: + """Validate meta indicator values in IamDataFrame. + + Parameters + ---------- + df : IamDataFrame + Input data whose meta indicators will be validated + **kwargs : dict + Validation criteria + + Returns + ------- + pd.DataFrame | None + A DataFrame of failing scenarios if any, otherwise None + + Raises + ------ + ValueError + *If a meta indicator in the 'df' is not listed in the .yaml + definition file + """ + value = kwargs.get("value") + upper_bound = kwargs.get("upper_bound") + lower_bound = kwargs.get("lower_bound") + if df.empty: + column_name = "', '".join(df.columns) + logger.warning( + f"Columns '{column_name}' do not exist in `meta`, skipping validation." + ) + return + _df = df.copy() + + failed_index = set() + if value is not None: + failed_index.update(_df[~_df.isin(value)].dropna(how="all").index) + if upper_bound is not None: + failed_index.update(_df[_df > upper_bound].dropna(how="all").index) + if lower_bound is not None: + failed_index.update(_df[_df < lower_bound].dropna(how="all").index) + if not failed_index: + return + _df = df.loc[sorted(failed_index)] + + if not _df.empty: + msg = "{} of {} meta indicators do not satisfy the criteria" + logger.warning(msg.format(len(_df), len(df))) + return _df diff --git a/nomenclature/processor/validator.py b/nomenclature/processor/validator.py new file mode 100644 index 00000000..82ee43ac --- /dev/null +++ b/nomenclature/processor/validator.py @@ -0,0 +1,297 @@ +import abc +import logging +import pandas as pd +from pathlib import Path +from enum import IntEnum +from pydantic import ( + BaseModel, + ConfigDict, + Field, + field_validator, + model_validator, + computed_field, +) +from pyam import IamDataFrame +from pyam.utils import adjust_log_level +from toolkit.exceptions import NoTracebackException +from nomenclature.codelist import CodeList +from nomenclature.definition import DataStructureDefinition +from nomenclature.processor.processor import Processor +from nomenclature.utils import get_relative_path + +logger = logging.getLogger(__name__) + + +class WarningEnum(IntEnum): + error = 50 + high = 40 + medium = 30 + low = 20 + + +class ValidationCriteria(abc.ABC, BaseModel): + """Base class for validation criteria (value, bounds, range)""" + + warning_level: WarningEnum = WarningEnum.error + + model_config = ConfigDict(extra="forbid") + + @field_validator("warning_level", mode="before") + @classmethod + def validate_warning_level(cls, value): + if isinstance(value, str): + try: + return WarningEnum[value] + except KeyError: + raise ValueError( + f"Invalid warning level: {value}. Expected one of:" + f" {', '.join(level.name for level in WarningEnum)}" + ) + return value + + @property + @abc.abstractmethod + def validation_args(self): + """Attributes used for validation.""" + pass + + @property + @abc.abstractmethod + def criteria(self): + """Attributes used for validation (as specified in the file).""" + pass + + def __str__(self): + return ", ".join([f"{key}: {value}" for key, value in self.criteria.items()]) + + +class ValidationValue(ValidationCriteria): + value: float + rtol: float = 0.0 + atol: float = 0.0 + + @property + def tolerance(self) -> float | None: + return ( + self.value * self.rtol + self.atol + if isinstance(self.value, float) + else None + ) + + @computed_field + @property + def upper_bound(self) -> float | None: + return self.value + self.tolerance if isinstance(self.value, float) else None + + @computed_field + @property + def lower_bound(self) -> float | None: + return self.value - self.tolerance if isinstance(self.value, float) else None + + @property + def validation_args(self): + # In case of list of values, validation is a hard equality check + if isinstance(self.value, list): + return {"value": self.value} + # Else, return the bounds for tolerance check + return self.model_dump( + exclude_none=True, + exclude_unset=True, + exclude=["warning_level", "value", "rtol", "atol"], + ) + + @property + def criteria(self): + return self.model_dump( + exclude_none=True, + exclude_unset=True, + exclude=["warning_level", "lower_bound", "upper_bound"], + ) + + +class ValidationBounds(ValidationCriteria): + upper_bound: float | None = None + lower_bound: float | None = None + + # Allow extra but raise error to guard against multiple criteria + model_config = ConfigDict(extra="allow") + + @model_validator(mode="after") + def check_validation_criteria_exist(self): + if self.upper_bound is None and self.lower_bound is None: + raise ValueError("No validation criteria provided: " + str(self.criteria)) + return self + + @model_validator(mode="after") + def check_validation_multiple_criteria(self): + if self.model_extra: + raise ValueError( + "Must use either bounds, range or value, found: " + str(self.criteria) + ) + return self + + @property + def validation_args(self): + return self.criteria + + @property + def criteria(self): + return self.model_dump( + exclude_none=True, exclude_unset=True, exclude=["warning_level"] + ) + + +class ValidationRange(ValidationCriteria): + range: list[float] = Field(..., min_length=2, max_length=2) + + @field_validator("range", mode="after") + @classmethod + def check_range_is_valid(cls, value: list[float | int]) -> list[float | int]: + if value[0] > value[1]: + raise ValueError( + "Validation 'range' must be given as `(lower_bound, upper_bound)`, " + "found: " + str(value) + ) + return value + + @computed_field + @property + def upper_bound(self) -> float: + return self.range[1] + + @computed_field + @property + def lower_bound(self) -> float: + return self.range[0] + + @property + def validation_args(self): + return self.model_dump( + exclude_none=True, + exclude_unset=True, + exclude=["warning_level", "range"], + ) + + @property + def criteria(self): + return self.model_dump( + exclude_none=True, + exclude_unset=True, + exclude=["warning_level", "lower_bound", "upper_bound"], + ) + + +class ValidationItem(BaseModel, abc.ABC): + """Base class for validation items (filter + criteria)""" + + name: str | None = None + validation: list[ValidationValue | ValidationBounds | ValidationRange] + + @model_validator(mode="after") + def check_warnings_order(self): + """Check if warnings are set in descending order of severity.""" + if self.validation != sorted( + self.validation, key=lambda c: c.warning_level, reverse=True + ): + raise ValueError( + f"Validation criteria for {self.criteria} not sorted" + " in descending order of severity." + ) + else: + return self + + @property + def filter_args(self): + return self.model_dump( + exclude_none=True, exclude_unset=True, exclude=["validation", "name"] + ) + + @abc.abstractmethod + def apply(self, df: IamDataFrame, fail_list: list, output_list: list): + """Apply validation to IamDataFrame.""" + pass + + def __str__(self): + return ", ".join([f"{key}: {value}" for key, value in self.filter_args.items()]) + + +class Validator(Processor): + """Abstract validation and processing class""" + + criteria_items: list[ValidationItem] + file: Path | str + output_path: Path | None = None + exception_cls: type[NoTracebackException] = NoTracebackException + + @classmethod + @abc.abstractmethod + def from_file( + cls, file: Path | str, output_path: Path | str | None = None + ) -> "Validator": + """Create a Validator instance from a file.""" + pass + + @classmethod + @abc.abstractmethod + def from_codelist( + cls, codelist: CodeList, output_path: Path | None = None + ) -> "Validator": + """Create a Validator from a CodeList""" + pass + + @abc.abstractmethod + def validate_with_definition(self, dsd: DataStructureDefinition) -> None: + """Validate the criteria items against a :class:`DataStructureDefinition`. + + Checks that all codes referenced in the criteria exist in the provided definition. + + Parameters + ---------- + dsd : DataStructureDefinition + Data structure definition to validate against. + + Raises + ------ + ExceptionGroup + If any criteria item references unknown codes. + """ + pass + + def apply(self, df: IamDataFrame) -> IamDataFrame: + """Validates data in IAMC format according to specified criteria. + + Logs warning/error messages for each criterion that is not met. + + Parameters + ---------- + df : pyam.IamDataFrame + Data in IAMC format to be validated + + Returns + ------- + pyam.IamDataFrame + + Raises + ------ + :exc:`ValueError` if any criterion has a warning level of ``error`` + """ + + error_list: list[bool] = [] + fail_list: list[str] = [] + output_list: list[pd.DataFrame] = [] + + with adjust_log_level(): + for item in self.criteria_items: + error, fail_list, output_list = item.apply(df, fail_list, output_list) + error_list.append(error) + if self.output_path: + pd.concat(output_list).to_excel(self.output_path, index=False) + fail_msg = f"(file {get_relative_path(self.file)}):\n" + if any(error_list): + raise self.exception_cls(fail_list, self.file) + if fail_list: + fail_msg = ( + "Data validation with warning(s) " + fail_msg + "\n".join(fail_list) + ) + logger.warning(fail_msg) + return df diff --git a/nomenclature/testing.py b/nomenclature/testing.py index d510b5e6..e34fc36a 100644 --- a/nomenclature/testing.py +++ b/nomenclature/testing.py @@ -12,9 +12,10 @@ ) from nomenclature.processor import ( DataValidator, + RequiredDataValidator, + MetaValidator, Processor, RegionProcessor, - RequiredDataValidator, ) logger = logging.getLogger(__name__) @@ -71,7 +72,7 @@ def _check_mappings( def _collect_processor_errors( path: Path, - processor: type[RequiredDataValidator] | type[DataValidator], + processor: type[RequiredDataValidator] | type[DataValidator] | type[MetaValidator], dsd: DataStructureDefinition, ) -> None: errors: list[NoTracebackExceptionGroup] = [] @@ -108,6 +109,7 @@ def assert_valid_structure( mappings: str | None = None, required_data: str | None = None, validate_data: str | None = None, + validate_meta: str | None = None, dimensions: list[str] | None = None, ) -> None: """Assert that `path` can be initialized as a :class:`DataStructureDefinition` @@ -126,6 +128,9 @@ def assert_valid_structure( validate_data : str, optional Name of the folder for data validation criteria, defaults to "validate_data" (if this folder exists) + validate_meta : str, optional + Name of the folder for meta validation criteria, defaults to "validate_meta" + (if this folder exists) dimensions : list[str], optional Dimensions to be checked, defaults to all sub-folders of `definitions` @@ -151,3 +156,4 @@ def assert_valid_structure( path, dsd, RequiredDataValidator, "required_data", required_data ) _check_processor_directory(path, dsd, DataValidator, "validate_data", validate_data) + _check_processor_directory(path, dsd, MetaValidator, "validate_meta", validate_meta) diff --git a/nomenclature/utils.py b/nomenclature/utils.py index 8b04dc22..af7faff7 100644 --- a/nomenclature/utils.py +++ b/nomenclature/utils.py @@ -23,3 +23,7 @@ def handle_remove_readonly(func, path, excinfo): func(path) else: raise + + +def single_input_to_list(v): + return v if isinstance(v, list) else [v] diff --git a/tests/data/meta_validator/definitions1/meta/meta_indicators_allowed_values.yaml b/tests/data/meta_validator/definitions/meta/meta_indicators_allowed_values.yaml similarity index 100% rename from tests/data/meta_validator/definitions1/meta/meta_indicators_allowed_values.yaml rename to tests/data/meta_validator/definitions/meta/meta_indicators_allowed_values.yaml diff --git a/tests/data/meta_validator/definitions2/meta/meta_indicators_test_data.yaml b/tests/data/meta_validator/definitions2/meta/meta_indicators_test_data.yaml deleted file mode 100644 index 62672440..00000000 --- a/tests/data/meta_validator/definitions2/meta/meta_indicators_test_data.yaml +++ /dev/null @@ -1,6 +0,0 @@ -- boolean: - allowed_values: [True, False] -- number: - allowed_values: [1.0, 2.0, 3.0, 4.0] -- string: - allowed_values: ['foo', 'bar'] \ No newline at end of file diff --git a/tests/data/meta_validator/definitions3/meta/meta_indicators_more_data.yaml b/tests/data/meta_validator/definitions3/meta/meta_indicators_more_data.yaml deleted file mode 100644 index fb2b1d4b..00000000 --- a/tests/data/meta_validator/definitions3/meta/meta_indicators_more_data.yaml +++ /dev/null @@ -1,6 +0,0 @@ -- meta_string: - allowed_values: ['A', 'B'] -- number: - allowed_values: [1.0, 2.0, 3.0, 4.0] -- string: - allowed_values: ['foo', 'bar'] diff --git a/tests/data/meta_validator/validate_meta/indicator_not_defined.yaml b/tests/data/meta_validator/validate_meta/indicator_not_defined.yaml new file mode 100644 index 00000000..c8ed68cc --- /dev/null +++ b/tests/data/meta_validator/validate_meta/indicator_not_defined.yaml @@ -0,0 +1,5 @@ +- name: Number Meta-Indicator Validation + meta: not defined + validation: + - warning_level: high + upper_bound: 1 diff --git a/tests/data/meta_validator/validate_meta/warning_error.yaml b/tests/data/meta_validator/validate_meta/warning_error.yaml new file mode 100644 index 00000000..b3f97901 --- /dev/null +++ b/tests/data/meta_validator/validate_meta/warning_error.yaml @@ -0,0 +1,3 @@ +- name: String Meta-Indicator Validation + meta: string + value: ["foo"] diff --git a/tests/data/meta_validator/validate_meta/warning_high.yaml b/tests/data/meta_validator/validate_meta/warning_high.yaml new file mode 100644 index 00000000..23d44f63 --- /dev/null +++ b/tests/data/meta_validator/validate_meta/warning_high.yaml @@ -0,0 +1,5 @@ +- name: Number Meta-Indicator Validation + meta: number + validation: + - warning_level: high + upper_bound: 1 diff --git a/tests/data/meta_validator/validate_meta/warning_multiple.yaml b/tests/data/meta_validator/validate_meta/warning_multiple.yaml new file mode 100644 index 00000000..8f4e66a6 --- /dev/null +++ b/tests/data/meta_validator/validate_meta/warning_multiple.yaml @@ -0,0 +1,11 @@ +- name: Number Meta-Indicator Validation + meta: number + validation: + - warning_level: high + upper_bound: 1 + - warning_level: medium + upper_bound: 0 +- name: String Meta-Indicator Validation + meta: string + warning_level: low + value: ["foo"] diff --git a/tests/data/meta_validator/validate_meta/warning_multiple_columns.yaml b/tests/data/meta_validator/validate_meta/warning_multiple_columns.yaml new file mode 100644 index 00000000..8db5843b --- /dev/null +++ b/tests/data/meta_validator/validate_meta/warning_multiple_columns.yaml @@ -0,0 +1,7 @@ +- name: Number Meta-Indicator Validation + meta: [number, number_too] + validation: + - warning_level: high + upper_bound: 1 + - warning_level: low + upper_bound: 0 diff --git a/tests/data/meta_validator/validate_meta/warning_value_tolerance.yaml b/tests/data/meta_validator/validate_meta/warning_value_tolerance.yaml new file mode 100644 index 00000000..287c204b --- /dev/null +++ b/tests/data/meta_validator/validate_meta/warning_value_tolerance.yaml @@ -0,0 +1,8 @@ +- name: Number Meta-Indicator Validation + meta: number + value: 1 + validation: + - warning_level: high + atol: 1 + - warning_level: medium + atol: 0.5 diff --git a/tests/test_code.py b/tests/test_code.py index 07ef508b..bcb16ca9 100644 --- a/tests/test_code.py +++ b/tests/test_code.py @@ -1,7 +1,7 @@ import pytest from pytest import raises -from nomenclature.code import Code, MetaCode, RegionCode, VariableCode +from nomenclature.code import Code, RegionCode, VariableCode def test_variable_without_unit_raises(): @@ -182,15 +182,6 @@ def test_RegionCode_iso3_code_str_fail(): RegionCode(name="Austria", hierarchy="country", iso3_codes="AUTT") -def test_MetaCode_allowed_values_attribute(): - meta = MetaCode( - name="MetaCode test", - allowed_values=[True], - ) - - assert meta.allowed_values == [True] - - def test_code_with_multi_key_dict_raises(): with raises(ValueError, match="Code is not a single name-attributes mapping"): Code.from_dict({"name": "", "illegal second key": ""}) diff --git a/tests/test_codelist.py b/tests/test_codelist.py index b76cf9f6..07c87537 100644 --- a/tests/test_codelist.py +++ b/tests/test_codelist.py @@ -523,12 +523,14 @@ def test_MetaCodeList_from_directory(): "exclude": MetaCode( name="exclude", description=None, - allowed_values=[True, False], + values=[True, False], + extra_attributes={"allowed_values": [True, False]}, ), "Meta cat with int values": MetaCode( name="Meta cat with int values", description=None, - allowed_values=[1, 2, 3], + values=[1, 2, 3], + extra_attributes={"allowed_values": [1, 2, 3]}, ), } exp = MetaCodeList(name="Meta", mapping=mapping) diff --git a/tests/test_config.py b/tests/test_config.py index ea636e23..f733337d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -111,9 +111,7 @@ def test_config_dimensions(): def test_invalid_config_dimensions_raises(): with raises( ValueError, - match=( - "Input should be 'model', .* 'region', 'subannual' or 'meta" - ), + match=("Input should be 'model', .* 'region', 'subannual' or 'meta"), ): NomenclatureConfig(dimensions=["year"]) diff --git a/tests/test_meta.py b/tests/test_meta.py index 352314b2..9a9e0eda 100644 --- a/tests/test_meta.py +++ b/tests/test_meta.py @@ -1,37 +1,228 @@ -import pyam import pytest +import pandas as pd +from pyam import IamDataFrame +from pyam.utils import IAMC_IDX +from nomenclature.processor.validator import WarningEnum +from nomenclature.codelist import MetaCodeList +from nomenclature.definition import DataStructureDefinition from nomenclature.processor.meta import MetaValidator +from nomenclature.exceptions import ( + NoTracebackException, + MetaValidationError, +) from conftest import TEST_DATA_DIR MODULE_TEST_DATA_DIR = TEST_DATA_DIR / "meta_validator" -def test_MetaValidator(simple_df): - meta_validator = MetaValidator(MODULE_TEST_DATA_DIR / "definitions1" / "meta") - exp = simple_df.copy() - pyam.testing.assert_iamframe_equal(exp, meta_validator.apply(df=simple_df)) +def test_MetaValidator_from_codelist(simple_df): + """ + Test MetaValidator can be created from a MetaCodeList and validation criteria + are set correctly (backwards-compatible with alias). + """ + meta_codelist = MetaCodeList.from_directory( + "meta", MODULE_TEST_DATA_DIR / "definitions" / "meta" + ) + meta_validator = MetaValidator.from_codelist(meta_codelist) + assert meta_validator.criteria_items[0].validation[0].value == [True, False] + assert meta_validator.criteria_items[1].validation[0].value == [1.0, 2.0, 3.0, 4.0] + assert meta_validator.criteria_items[2].validation[0].value == ["foo", "bar"] + + +def test_MetaValidator_from_file(): + """ + Test MetaValidator can be created from a YAML file and validation criteria + are set correctly. + """ + meta_validator = MetaValidator.from_file( + MODULE_TEST_DATA_DIR / "validate_meta" / "warning_multiple.yaml" + ) + assert meta_validator.criteria_items[0].validation[0].upper_bound == 1.0 + assert ( + meta_validator.criteria_items[0].validation[0].warning_level == WarningEnum.high + ) + assert meta_validator.criteria_items[0].validation[1].upper_bound == 0.0 + assert ( + meta_validator.criteria_items[0].validation[1].warning_level + == WarningEnum.medium + ) + assert meta_validator.criteria_items[1].validation[0].value == ["foo"] + assert ( + meta_validator.criteria_items[1].validation[0].warning_level == WarningEnum.low + ) + + +def test_MetaValidator_validate_with_definition(): + """ + Test MetaValidator's criteria items against the MetaCodeList. + """ + meta_codelist = MetaCodeList.from_directory( + "meta", MODULE_TEST_DATA_DIR / "definitions" / "meta" + ) + meta_validator = MetaValidator.from_codelist(meta_codelist) + dsd = DataStructureDefinition(MODULE_TEST_DATA_DIR / "definitions") + + assert meta_validator.validate_with_definition(dsd) is None + + +def test_MetaValidator_validate_with_definition_raises(): + """ + Test MetaValidator's DSD validation when criteria uses indicators not in definition. + """ + error_msg = ( + "The following meta indicators are not defined " + "in the DataStructureDefinition:\n 'not defined'" + ) + + meta_validator = MetaValidator.from_file( + MODULE_TEST_DATA_DIR / "validate_meta" / "indicator_not_defined.yaml" + ) + dsd = DataStructureDefinition(MODULE_TEST_DATA_DIR / "definitions") + + with pytest.RaisesGroup(NoTracebackException) as excinfo: + meta_validator.validate_with_definition(dsd) + assert excinfo.group_contains(NoTracebackException, match=error_msg) + + +def test_MetaValidator_apply_warning(simple_df, caplog): + """ + Test MetaValidator's criteria items against a data frame. + """ + warning_msg = """ Criteria: meta: ['number'], upper_bound: 1.0 + number warning_level + model scenario + model_a scen_b 2.0 high""" + + meta_validator = MetaValidator.from_file( + MODULE_TEST_DATA_DIR / "validate_meta" / "warning_high.yaml" + ) + meta_validator.apply(simple_df) + assert warning_msg in caplog.text + + +def test_MetaValidator_apply_multiple_warning_levels(simple_df, caplog): + """ + Test MetaValidator can apply multiple warning levels to meta indicators. + """ + warning_msg = """ + Criteria: meta: ['number'], upper_bound: 1.0 + number warning_level + model scenario + model_a scen_b 2.0 high + Criteria: meta: ['number'], upper_bound: 0.0 + number warning_level + model scenario + model_a scen_a 1.0 medium -def test_MetaValidator_Meta_Indicator_Error(simple_df): - simple_df.set_meta(name="not allowed", meta=False) - meta_validator = MetaValidator(MODULE_TEST_DATA_DIR / "definitions2" / "meta") - match = ( - "Invalid meta indicator: 'not allowed'\n" # noqa - "Valid meta indicators: 'boolean', 'number', 'string'" # noqa + Criteria: meta: ['string'], value: ['foo'] + string warning_level + model scenario + model_a scen_b bar low""" + + meta_validator = MetaValidator.from_file( + MODULE_TEST_DATA_DIR / "validate_meta" / "warning_multiple.yaml" + ) + meta_validator.apply(simple_df) + assert warning_msg in caplog.text + + +def test_MetaValidator_apply_value_tolerance(simple_df, caplog): + """ + Test MetaValidator allows validation with value and tolerance for `value` field. + """ + warning_msg = """ Criteria: meta: ['number'], value: 1.0, atol: 0.5 + number warning_level + model scenario + model_a scen_b 2.0 medium""" + + meta_validator = MetaValidator.from_file( + MODULE_TEST_DATA_DIR / "validate_meta" / "warning_value_tolerance.yaml" + ) + meta_validator.apply(simple_df) + assert warning_msg in caplog.text + + +def test_MetaValidator_apply_multiple_columns(simple_df, caplog): + """ + Test MetaValidator allows simultaneous validation for multiple meta columns. + Higher-level warnings are prioritised over lower-level warnings for the same scenario. + """ + warning_msg = """ Criteria: meta: ['number', 'number_too'], upper_bound: 1.0 + number number_too warning_level + model scenario + model_a scen_a 1.0 2.0 high + scen_b 2.0 1.0 high""" + simple_df.set_meta([2.0, 1.0], "number_too") + warning_msg = """""" + meta_validator = MetaValidator.from_file( + MODULE_TEST_DATA_DIR / "validate_meta" / "warning_multiple_columns.yaml" + ) + meta_validator.apply(simple_df) + assert warning_msg in caplog.text + assert "upper_bound: 0.0" not in caplog.text + assert "low" not in caplog.text + + +def test_MetaValidator_apply_empty_df(caplog): + """ + Test MetaValidator on an empty data frame (columns but no rows). + """ + empty_df = IamDataFrame(pd.DataFrame([], columns=IAMC_IDX + [2005, 2010])) + empty_df.set_meta([], "number") + + meta_validator = MetaValidator.from_file( + MODULE_TEST_DATA_DIR / "validate_meta" / "warning_high.yaml" ) + meta_validator.apply(empty_df) + + assert ( + "Columns 'number' do not exist in `meta`, skipping validation." in caplog.text + ) + + empty_df.set_meta([], "number_too") + meta_validator = MetaValidator.from_file( + MODULE_TEST_DATA_DIR / "validate_meta" / "warning_multiple_columns.yaml" + ) + meta_validator.apply(empty_df) + + assert ( + "Columns 'number', 'number_too' do not exist in `meta`, skipping validation." + in caplog.text + ) + + +def test_MetaValidator_apply_ignore_missing_column(simple_df, caplog): + """ + Test MetaValidator on a data frame with a missing meta indicator . + """ + warning_msg = """ Criteria: meta: ['number', 'number_too'], upper_bound: 1.0 + number warning_level + model scenario + model_a scen_b 2.0 high""" + + meta_validator = MetaValidator.from_file( + MODULE_TEST_DATA_DIR / "validate_meta" / "warning_multiple_columns.yaml" + ) + meta_validator.apply(simple_df) + assert warning_msg in caplog.text + - with pytest.raises(ValueError, match=match): - meta_validator.apply(df=simple_df) +def test_MetaValidator_apply_error(simple_df): + """ + Test MetaValidator's criteria items against a data frame. + """ + error_msg = """Criteria: meta: ['string'], value: ['foo'] + string warning_level + model scenario + model_a scen_b bar error""" -def test_MetaValidator_Meta_Indicator_Value_Error(simple_df): - simple_df.set_meta(name="meta_string", meta=3) - meta_validator = MetaValidator(MODULE_TEST_DATA_DIR / "definitions3" / "meta") - match = ( - "Invalid value for meta indicator 'meta_string': '3'\n" # noqa - "Allowed values: 'A', 'B'" # noqa + meta_validator = MetaValidator.from_file( + MODULE_TEST_DATA_DIR / "validate_meta" / "warning_error.yaml" ) - with pytest.raises(ValueError, match=match): - meta_validator.apply(df=simple_df) + with pytest.raises(MetaValidationError) as excinfo: + meta_validator.apply(simple_df) + assert error_msg in str(excinfo.value) diff --git a/tests/test_validate_data.py b/tests/test_validate_data.py index c2752416..1637de8d 100644 --- a/tests/test_validate_data.py +++ b/tests/test_validate_data.py @@ -11,7 +11,7 @@ from nomenclature import DataStructureDefinition from nomenclature.codelist import VariableCodeList from nomenclature.exceptions import DataValidationError -from nomenclature.processor.data_validator import DataValidator +from nomenclature.processor.data import DataValidator DATA_VALIDATION_TEST_DIR = TEST_DATA_DIR / "validation" / "validate_data" PROCESSOR_TEST_DIR = TEST_DATA_DIR / "processor" / "data_validator"