diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ccd46ba..9f754930 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Updated initial_config-relecov.yaml and added comments for some params [#875](https://github.com/BU-ISCIII/relecov-tools/pull/875) - Fixed new version filename taxprofiler[#877](https://github.com/BU-ISCIII/relecov-tools/pull/877) -- Fix duplicated row in metadata form and clean some MEPRAM initial config [#879] https://github.com/BU-ISCIII/relecov-tools/pull/879 +- Fix duplicated row in metadata form and clean some MEPRAM initial config [#879] (https://github.com/BU-ISCIII/relecov-tools/pull/879) +- Fix date validation in Excel metadata templates and add some schema validation checks [#886] (https://github.com/BU-ISCIII/relecov-tools/pull/886) #### Changed diff --git a/relecov_tools/assets/schema_utils/metadatalab_template.py b/relecov_tools/assets/schema_utils/metadatalab_template.py index 360393fa..af6c2fbd 100644 --- a/relecov_tools/assets/schema_utils/metadatalab_template.py +++ b/relecov_tools/assets/schema_utils/metadatalab_template.py @@ -273,7 +273,14 @@ def create_condition(ws_metadata, conditions, df_filtered): ws_metadata.add_data_validation(validation) validation.add(cell_range) - if rules.get("format_cells_as_date", False): + if rules.get("format_cells_as_text", False): + col_idx = column_index_from_string(col_letter) + for row in ws_metadata.iter_rows( + min_row=start_row, max_row=end_row, min_col=col_idx, max_col=col_idx + ): + for cell in row: + cell.number_format = "@" + elif rules.get("format_cells_as_date", False): col_idx = column_index_from_string(col_letter) for row in ws_metadata.iter_rows( min_row=start_row, max_row=end_row, min_col=col_idx, max_col=col_idx diff --git a/relecov_tools/build_schema.py b/relecov_tools/build_schema.py index 31153803..3c14cee2 100644 --- a/relecov_tools/build_schema.py +++ b/relecov_tools/build_schema.py @@ -257,7 +257,9 @@ def _load_laboratory_addresses(self): dropdowns[f].append(dropdown_entry) uniques[f].add(name) - dropdowns = {k: sorted(v) for k, v in dropdowns.items()} + dropdowns = { + k: sorted(self._unique_enum_values(v)) for k, v in dropdowns.items() + } uniques = {k: sorted(v) for k, v in uniques.items()} return dropdowns, uniques @@ -311,6 +313,7 @@ def validate_database_definition(self, json_data): - Missing features - Duplicate enums - Missing examples + - Invalid enum examples - Invalid example types - Incorrect date formats """ @@ -318,6 +321,7 @@ def validate_database_definition(self, json_data): "missing_features": {}, "missing_examples": {}, "duplicate_enums": {}, + "invalid_enum_examples": {}, "invalid_example_types": {}, "invalid_date_formats": {}, } @@ -364,6 +368,17 @@ def validate_database_definition(self, json_data): log_errors["missing_examples"][prop_name] = ["Missing example."] feature_type = prop_features["type"] + + # Check examples are included in the enum values, when an enum exists. + invalid_enum_examples = self._validate_examples_in_enum( + prop_name, + example, + prop_features.get("enum"), + feature_type, + ) + if invalid_enum_examples: + log_errors["invalid_enum_examples"][prop_name] = invalid_enum_examples + match feature_type: # Check date format for properties with type=string and format=date case "string": @@ -432,6 +447,62 @@ def validate_database_definition(self, json_data): # If no errors found return None + def _validate_examples_in_enum( + self, + property_id: str, + example_value: any, + enum_value: any, + expected_type: str | None, + ) -> list[str]: + """Return validation errors for examples that are not present in enum.""" + if self._is_empty_validation_value(enum_value): + return [] + if self._is_empty_validation_value(example_value): + return [] + + enum_values = self._parse_enum_values(enum_value) + if not isinstance(enum_values, list) or not enum_values: + return [] + + examples = self._parse_examples_for_validation(example_value) + examples = self._cast_examples_to_declared_type( + property_id, expected_type, examples + ) + + enum_lookup = { + self._normalize_enum_example_value(value) for value in enum_values + } + return [ + f"Example '{example}' is not defined in enum." + for example in examples + if self._normalize_enum_example_value(example) not in enum_lookup + ] + + @staticmethod + def _normalize_enum_example_value(value: any) -> any: + if not isinstance(value, str): + return value + return re.sub(r"\s*\[[^\]]+\]$", "", value).strip() + + @staticmethod + def _is_empty_validation_value(value: any) -> bool: + if value is None or isinstance(value, list): + return value is None + return pd.isna(value) + + @staticmethod + def _parse_examples_for_validation(example_value: any) -> list[any]: + """Parse the examples cell using the same separator used for schema examples.""" + if isinstance(example_value, str): + return [ + value.strip() for value in example_value.split("; ") if value.strip() + ] + if isinstance(example_value, datetime): + return [example_value.strftime("%Y-%m-%d")] + if isinstance(example_value, float) and example_value.is_integer(): + return [int(example_value)] + return [example_value] + def get_available_projects(self, json): """Get list of available software in configuration @@ -841,10 +912,70 @@ def verify_schema(self, schema): Raises: ValueError: If the schema does not conform to the JSON Schema specification. """ + self.validate_schema_enum_duplicates(schema) relecov_tools.assets.schema_utils.jsonschema_draft.check_schema_draft( schema, self.draft_version ) + @staticmethod + def _find_duplicate_values(values: list) -> list: + """Return duplicated values preserving first duplicate encounter order.""" + seen = set() + duplicates = [] + duplicate_seen = set() + for value in values: + lookup_value = value.strip() if isinstance(value, str) else value + try: + is_seen = lookup_value in seen + except TypeError: + lookup_value = json.dumps(value, sort_keys=True, ensure_ascii=False) + is_seen = lookup_value in seen + + if is_seen and lookup_value not in duplicate_seen: + duplicates.append(value) + duplicate_seen.add(lookup_value) + else: + seen.add(lookup_value) + return duplicates + + def validate_schema_enum_duplicates(self, schema: dict): + """Validate that every enum list in a generated schema has unique values.""" + duplicate_enums = {} + + def walk_schema(node, path="$"): + if isinstance(node, dict): + enum_values = node.get("enum") + if isinstance(enum_values, list): + duplicates = self._find_duplicate_values(enum_values) + if duplicates: + duplicate_enums[path] = duplicates + for key, value in node.items(): + walk_schema(value, f"{path}.{key}") + elif isinstance(node, list): + for index, value in enumerate(node): + walk_schema(value, f"{path}[{index}]") + + walk_schema(schema) + + if duplicate_enums: + df_errors = pd.DataFrame( + [ + { + "Enum Path": enum_path, + "Duplicate Values": "; ".join(map(str, duplicates)), + } + for enum_path, duplicates in duplicate_enums.items() + ] + ) + error_file_path = f"{self.output_dir}/schema_enum_duplicates.csv" + df_errors.to_csv(error_file_path, index=False, encoding="utf-8") + relecov_tools.utils.display_dataframe_to_user( + name="Schema Enum Duplicates", dataframe=df_errors + ) + stderr.print("[red]Duplicated enum values found. Log saved to:") + stderr.print(f"\t{error_file_path}") + raise ValueError("Duplicated enum values found in generated schema.") + def get_schema_diff(self, base_schema, new_schema): """ Print the differences between the base schema and the newly generated schema. @@ -945,7 +1076,7 @@ def save_new_schema(self, json_data): def _sort_enum_values(self, enum_values: list[str]) -> list[str]: """ - Sort enum values alphabetically while keeping configured special terms last. + Deduplicate and sort enum values alphabetically while keeping configured special terms last. Args: enum_values (list[str]): Enum values to sort. @@ -964,6 +1095,8 @@ def _sort_enum_values(self, enum_values: list[str]) -> list[str]: "other": 7, } + unique_values = self._unique_enum_values(enum_values) + def sort_key(value: str): normalized_value = value.strip().casefold() for last_value, last_order in last_values.items(): @@ -971,7 +1104,24 @@ def sort_key(value: str): return (1, last_order) return (0, normalized_value) - return sorted(enum_values, key=sort_key) + return sorted(unique_values, key=sort_key) + + @staticmethod + def _unique_enum_values(enum_values: list) -> list: + """Return enum values without duplicates, preserving first occurrence order.""" + unique_values = [] + seen = set() + for value in enum_values: + lookup_value = value.strip() if isinstance(value, str) else value + try: + is_seen = lookup_value in seen + except TypeError: + lookup_value = json.dumps(value, sort_keys=True, ensure_ascii=False) + is_seen = lookup_value in seen + if not is_seen: + unique_values.append(value) + seen.add(lookup_value) + return unique_values def _parse_enum_values(self, enum_value): """ @@ -1092,7 +1242,7 @@ def _load_enum_values_from_sheet(self, enum_value: str) -> list[str]: for value in df[column_name].dropna().tolist() if str(value).strip() ] - return list(dict.fromkeys(values)) + return values def _load_enum_values_from_file(self, enum_value: str) -> list[str]: """ @@ -1286,7 +1436,9 @@ def create_metadatalab_excel(self, json_schema, database_definition=None): ) def clean_ontologies(enums): - return [re.sub(r"\s*\[.*?\]", "", item).strip() for item in enums] + return self._unique_enum_values( + [re.sub(r"\s*\[.*?\]", "", item).strip() for item in enums] + ) def resolve_enum_ref(ref: str, enum_defs: dict) -> list[str]: property_key = ref.split("enums/")[-1] diff --git a/relecov_tools/conf/build_schema_config.json b/relecov_tools/conf/build_schema_config.json index d1ad0c2e..858af8ab 100644 --- a/relecov_tools/conf/build_schema_config.json +++ b/relecov_tools/conf/build_schema_config.json @@ -2,7 +2,7 @@ "projects" : { "relecov": { "host_age_years": { - "header_row_idx": 1, + "header_row_idx": 5, "max_rows": 1000, "validation_type": "whole", "operator": "between", @@ -12,7 +12,7 @@ "error_title": "Valor no permitido" }, "host_age_months": { - "header_row_idx": 1, + "header_row_idx": 5, "max_rows": 1000, "validation_type": "whole", "operator": "between", @@ -22,36 +22,36 @@ "error_title": "Valor no permitido" }, "sample_collection_date": { - "header_row_idx": 1, + "header_row_idx": 5, "max_rows": 1000, "validation_type": "custom", - "formula1": "=ISNUMBER({col_letter}5)", + "formula1": "=OR({col_letter}5=\"\",IFERROR(AND(ISTEXT({col_letter}5),LEN({col_letter}5)=10,MID({col_letter}5,5,1)=\"-\",MID({col_letter}5,8,1)=\"-\",VALUE(MID({col_letter}5,6,2))>=1,VALUE(MID({col_letter}5,6,2))<=12,VALUE(RIGHT({col_letter}5,2))>=1,VALUE(RIGHT({col_letter}5,2))<=31,ISNUMBER(DATEVALUE({col_letter}5))),FALSE))", "error_message": "Ingrese la fecha en formato correcto YYYY-MM-DD (ejemplo: 2024-02-12).", "error_title": "Formato de fecha incorrecto", - "format_cells_as_date": true + "format_cells_as_text": true }, "sample_received_date": { - "header_row_idx": 1, + "header_row_idx": 5, "max_rows": 1000, "validation_type": "custom", - "formula1": "=ISNUMBER({col_letter}5)", + "formula1": "=OR({col_letter}5=\"\",IFERROR(AND(ISTEXT({col_letter}5),LEN({col_letter}5)=10,MID({col_letter}5,5,1)=\"-\",MID({col_letter}5,8,1)=\"-\",VALUE(MID({col_letter}5,6,2))>=1,VALUE(MID({col_letter}5,6,2))<=12,VALUE(RIGHT({col_letter}5,2))>=1,VALUE(RIGHT({col_letter}5,2))<=31,ISNUMBER(DATEVALUE({col_letter}5))),FALSE))", "error_message": "Ingrese la fecha en formato correcto YYYY-MM-DD (ejemplo: 2024-02-12).", "error_title": "Formato de fecha incorrecto", - "format_cells_as_date": true + "format_cells_as_text": true }, "sequencing_date": { - "header_row_idx": 1, + "header_row_idx": 5, "max_rows": 1000, "validation_type": "custom", - "formula1": "=ISNUMBER({col_letter}5)", + "formula1": "=OR({col_letter}5=\"\",IFERROR(AND(ISTEXT({col_letter}5),LEN({col_letter}5)=10,MID({col_letter}5,5,1)=\"-\",MID({col_letter}5,8,1)=\"-\",VALUE(MID({col_letter}5,6,2))>=1,VALUE(MID({col_letter}5,6,2))<=12,VALUE(RIGHT({col_letter}5,2))>=1,VALUE(RIGHT({col_letter}5,2))<=31,ISNUMBER(DATEVALUE({col_letter}5))),FALSE))", "error_message": "Ingrese la fecha en formato correcto YYYY-MM-DD (ejemplo: 2024-02-12).", "error_title": "Formato de fecha incorrecto", - "format_cells_as_date": true + "format_cells_as_text": true } }, "mepram": { "host_age_years": { - "header_row_idx": 1, + "header_row_idx": 5, "max_rows": 1000, "validation_type": "whole", "operator": "between", @@ -61,7 +61,7 @@ "error_title": "Valor no permitido" }, "host_age_months": { - "header_row_idx": 1, + "header_row_idx": 5, "max_rows": 1000, "validation_type": "whole", "operator": "between", @@ -71,22 +71,22 @@ "error_title": "Valor no permitido" }, "patient_birth_date": { - "header_row_idx": 1, + "header_row_idx": 5, "max_rows": 1000, "validation_type": "custom", - "formula1": "=ISNUMBER({col_letter}5)", + "formula1": "=OR({col_letter}5=\"\",IFERROR(AND(ISTEXT({col_letter}5),LEN({col_letter}5)=10,MID({col_letter}5,5,1)=\"-\",MID({col_letter}5,8,1)=\"-\",VALUE(MID({col_letter}5,6,2))>=1,VALUE(MID({col_letter}5,6,2))<=12,VALUE(RIGHT({col_letter}5,2))>=1,VALUE(RIGHT({col_letter}5,2))<=31,ISNUMBER(DATEVALUE({col_letter}5))),FALSE))", "error_message": "Ingrese la fecha en formato correcto YYYY-MM-DD (ejemplo: 2024-02-12).", "error_title": "Formato de fecha incorrecto", - "format_cells_as_date": true + "format_cells_as_text": true }, "sample_collection_date": { - "header_row_idx": 1, + "header_row_idx": 5, "max_rows": 1000, "validation_type": "custom", - "formula1": "=ISNUMBER({col_letter}5)", + "formula1": "=OR({col_letter}5=\"\",IFERROR(AND(ISTEXT({col_letter}5),LEN({col_letter}5)=10,MID({col_letter}5,5,1)=\"-\",MID({col_letter}5,8,1)=\"-\",VALUE(MID({col_letter}5,6,2))>=1,VALUE(MID({col_letter}5,6,2))<=12,VALUE(RIGHT({col_letter}5,2))>=1,VALUE(RIGHT({col_letter}5,2))<=31,ISNUMBER(DATEVALUE({col_letter}5))),FALSE))", "error_message": "Ingrese la fecha en formato correcto YYYY-MM-DD (ejemplo: 2024-02-12).", "error_title": "Formato de fecha incorrecto", - "format_cells_as_date": true + "format_cells_as_text": true } } },