diff --git a/nomenclature/processor/data_validator.py b/nomenclature/processor/data_validator.py index c91fe846..0ceff134 100644 --- a/nomenclature/processor/data_validator.py +++ b/nomenclature/processor/data_validator.py @@ -388,3 +388,24 @@ def validate_with_definition(self, dsd: DataStructureDefinition) -> None: f"Error in DataValidator (file {get_relative_path(self.file)})", errors, ) + + @property + def input_data(self): + return merge_criteria_items_filter_args(self.criteria_items) + + +def merge_criteria_items_filter_args(list_of_items: list[DataValidationItem]) -> dict: + """Concatenate filter-arguments""" + merged_dict = {} + for dct in list_of_items: + for key, value in dct.filter_args.items(): + if key in merged_dict: + merged_dict[key].extend(value) + else: + merged_dict[key] = value + + # drop duplicates + for key, value in merged_dict.items(): + merged_dict[key] = list(set(merged_dict[key])) + + return merged_dict diff --git a/nomenclature/processor/processor.py b/nomenclature/processor/processor.py index 512172c3..066dec4c 100644 --- a/nomenclature/processor/processor.py +++ b/nomenclature/processor/processor.py @@ -6,12 +6,16 @@ class Processor(BaseModel, abc.ABC): - input_data: Annotated[dict[str, list[str]] | None, Field(frozen=True)] = None +# input_data: Annotated[dict[str, list[str]] | None, Field(frozen=True)] = None input_meta: Annotated[list[str] | None, Field(frozen=True)] = None output_data: Annotated[dict[str, list[str]] | None, Field(frozen=True)] = None output_meta: Annotated[list[str] | None, Field(frozen=True)] = None fail_ok: bool = False + @property + def input_meta(self) -> Annotated[list[str] | None, Field(frozen=True)]: + return + @abc.abstractmethod def apply(self, df: IamDataFrame) -> IamDataFrame: raise NotImplementedError diff --git a/tests/test_aggregator.py b/tests/test_aggregator.py index a2a419e0..c0323997 100644 --- a/tests/test_aggregator.py +++ b/tests/test_aggregator.py @@ -34,8 +34,7 @@ def test_aggregator_from_file(): "rename": [ {"name": "Resource|Extraction|Petrol", "rename": "Resource|Extraction|Oil"} ], - "input_data": None, - "input_meta": None, +# "input_data": None, "output_data": None, "output_meta": None, "fail_ok": False, diff --git a/tests/test_processor.py b/tests/test_processor.py index b3662be1..a07c9d8e 100644 --- a/tests/test_processor.py +++ b/tests/test_processor.py @@ -10,22 +10,22 @@ class ProcessorSubclass(Processor): def apply(self, df: IamDataFrame) -> IamDataFrame: return df - input_data = {"variable": ["Emissions|CO2"], "region": ["World"]} +# input_data = {"variable": ["Emissions|CO2"], "region": ["World"]} output_meta = ["Emissions Diagnostics|Year of Net Zero|CO2"] processor = ProcessorSubclass( - input_data=input_data, +# input_data=input_data, output_meta=output_meta, fail_ok=True, ) - assert processor.input_data == input_data +# assert processor.input_data == input_data assert processor.input_meta is None assert processor.output_data is None assert processor.output_meta == output_meta # check that frozen fields cannot be modified after instantiation - with pytest.raises(ValidationError, match="Field is frozen"): - processor.input_data = {"variable": ["Emissions|Kyoto Gases"]} - with pytest.raises(ValidationError, match="Field is frozen"): - processor.input_meta = ["Emissions Diagnostics|Cumulative CO2 [2020-2100"] +# with pytest.raises(ValidationError, match="Field is frozen"): +# processor.input_data = {"variable": ["Emissions|Kyoto Gases"]} +# with pytest.raises(ValidationError, match="Field is frozen"): +# processor.input_meta = ["Emissions Diagnostics|Cumulative CO2 [2020-2100"] assert processor.fail_ok is True diff --git a/tests/test_validate_data.py b/tests/test_validate_data.py index c2752416..4ae915da 100644 --- a/tests/test_validate_data.py +++ b/tests/test_validate_data.py @@ -42,6 +42,8 @@ def test_DataValidator_simple_from_file(): dsd = DataStructureDefinition(TEST_DATA_DIR / "validation" / "definitions") assert obs.validate_with_definition(dsd) is None + assert obs.input_data == {"variable": ["Final Energy"], "year": [2010]} + def test_DataValidator_structured_from_file(): exp = DataValidator( @@ -98,6 +100,10 @@ def test_DataValidator_from_codelist(): ) assert variable_codelist.data_validator == exp + assert variable_codelist.data_validator.input_data == { + "variable": ["Final Energy", "Final Energy|Electricity [Share]"] + } + @pytest.mark.parametrize( "name, match",