Fix date validation in Excel metadata templates and add some schema validation checks

CHANGELOG.md

@@ -23,7 +23,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 - Updated initial_config-relecov.yaml and added comments for some params [#875](https://github.com/BU-ISCIII/relecov-tools/pull/875)
 - Fixed new version filename taxprofiler[#877](https://github.com/BU-ISCIII/relecov-tools/pull/877)
-- Fix duplicated row in metadata form and clean some MEPRAM initial config [#879] https://github.com/BU-ISCIII/relecov-tools/pull/879
+- Fix duplicated row in metadata form and clean some MEPRAM initial config [#879] (https://github.com/BU-ISCIII/relecov-tools/pull/879)
+- Fix date validation in Excel metadata templates and add some schema validation checks [#886] (https://github.com/BU-ISCIII/relecov-tools/pull/886)
 
 #### Changed
 

relecov_tools/assets/schema_utils/metadatalab_template.py

@@ -273,7 +273,14 @@ def create_condition(ws_metadata, conditions, df_filtered):
                 ws_metadata.add_data_validation(validation)
                 validation.add(cell_range)
 
-            if rules.get("format_cells_as_date", False):
+            if rules.get("format_cells_as_text", False):
+                col_idx = column_index_from_string(col_letter)
+                for row in ws_metadata.iter_rows(
+                    min_row=start_row, max_row=end_row, min_col=col_idx, max_col=col_idx
+                ):
+                    for cell in row:
+                        cell.number_format = "@"
+            elif rules.get("format_cells_as_date", False):
                 col_idx = column_index_from_string(col_letter)
                 for row in ws_metadata.iter_rows(
                     min_row=start_row, max_row=end_row, min_col=col_idx, max_col=col_idx

relecov_tools/build_schema.py

@@ -257,7 +257,9 @@ def _load_laboratory_addresses(self):
                 dropdowns[f].append(dropdown_entry)
                 uniques[f].add(name)
 
-        dropdowns = {k: sorted(v) for k, v in dropdowns.items()}
+        dropdowns = {
+            k: sorted(self._unique_enum_values(v)) for k, v in dropdowns.items()
+        }
         uniques = {k: sorted(v) for k, v in uniques.items()}
         return dropdowns, uniques
 
@@ -311,13 +313,15 @@ def validate_database_definition(self, json_data):
                 - Missing features
                 - Duplicate enums
                 - Missing examples
+                - Invalid enum examples
                 - Invalid example types
                 - Incorrect date formats
         """
         log_errors = {
             "missing_features": {},
             "missing_examples": {},
             "duplicate_enums": {},
+            "invalid_enum_examples": {},
             "invalid_example_types": {},
             "invalid_date_formats": {},
         }
@@ -364,6 +368,17 @@ def validate_database_definition(self, json_data):
                 log_errors["missing_examples"][prop_name] = ["Missing example."]
 
             feature_type = prop_features["type"]
+
+            # Check examples are included in the enum values, when an enum exists.
+            invalid_enum_examples = self._validate_examples_in_enum(
+                prop_name,
+                example,
+                prop_features.get("enum"),
+                feature_type,
+            )
+            if invalid_enum_examples:
+                log_errors["invalid_enum_examples"][prop_name] = invalid_enum_examples
+
             match feature_type:
                 # Check date format for properties with type=string and format=date
                 case "string":
@@ -432,6 +447,62 @@ def validate_database_definition(self, json_data):
         # If no errors found
         return None
 
+    def _validate_examples_in_enum(
+        self,
+        property_id: str,
+        example_value: any,
+        enum_value: any,
+        expected_type: str | None,
+    ) -> list[str]:
+        """Return validation errors for examples that are not present in enum."""
+        if self._is_empty_validation_value(enum_value):
+            return []
+        if self._is_empty_validation_value(example_value):
+            return []
+
+        enum_values = self._parse_enum_values(enum_value)
+        if not isinstance(enum_values, list) or not enum_values:
+            return []
+
+        examples = self._parse_examples_for_validation(example_value)
+        examples = self._cast_examples_to_declared_type(
+            property_id, expected_type, examples
+        )
+
+        enum_lookup = {
+            self._normalize_enum_example_value(value) for value in enum_values
+        }
+        return [
+            f"Example '{example}' is not defined in enum."
+            for example in examples
+            if self._normalize_enum_example_value(example) not in enum_lookup
+        ]
+
+    @staticmethod
+    def _normalize_enum_example_value(value: any) -> any:
+        if not isinstance(value, str):
+            return value
+        return re.sub(r"\s*\[[^\]]+\]$", "", value).strip()
+
+    @staticmethod
+    def _is_empty_validation_value(value: any) -> bool:
+        if value is None or isinstance(value, list):
+            return value is None
+        return pd.isna(value)
+
+    @staticmethod
+    def _parse_examples_for_validation(example_value: any) -> list[any]:
+        """Parse the examples cell using the same separator used for schema examples."""
+        if isinstance(example_value, str):
+            return [
+                value.strip() for value in example_value.split("; ") if value.strip()
+            ]
+        if isinstance(example_value, datetime):
+            return [example_value.strftime("%Y-%m-%d")]
+        if isinstance(example_value, float) and example_value.is_integer():
+            return [int(example_value)]
+        return [example_value]
+
     def get_available_projects(self, json):
         """Get list of available software in configuration
 
@@ -841,10 +912,70 @@ def verify_schema(self, schema):
         Raises:
             ValueError: If the schema does not conform to the JSON Schema specification.
         """
+        self.validate_schema_enum_duplicates(schema)
         relecov_tools.assets.schema_utils.jsonschema_draft.check_schema_draft(
             schema, self.draft_version
         )
 
+    @staticmethod
+    def _find_duplicate_values(values: list) -> list:
+        """Return duplicated values preserving first duplicate encounter order."""
+        seen = set()
+        duplicates = []
+        duplicate_seen = set()
+        for value in values:
+            lookup_value = value.strip() if isinstance(value, str) else value
+            try:
+                is_seen = lookup_value in seen
+            except TypeError:
+                lookup_value = json.dumps(value, sort_keys=True, ensure_ascii=False)
+                is_seen = lookup_value in seen
+
+            if is_seen and lookup_value not in duplicate_seen:
+                duplicates.append(value)
+                duplicate_seen.add(lookup_value)
+            else:
+                seen.add(lookup_value)
+        return duplicates
+
+    def validate_schema_enum_duplicates(self, schema: dict):
+        """Validate that every enum list in a generated schema has unique values."""
+        duplicate_enums = {}
+
+        def walk_schema(node, path="$"):
+            if isinstance(node, dict):
+                enum_values = node.get("enum")
+                if isinstance(enum_values, list):
+                    duplicates = self._find_duplicate_values(enum_values)
+                    if duplicates:
+                        duplicate_enums[path] = duplicates
+                for key, value in node.items():
+                    walk_schema(value, f"{path}.{key}")
+            elif isinstance(node, list):
+                for index, value in enumerate(node):
+                    walk_schema(value, f"{path}[{index}]")
+
+        walk_schema(schema)
+
+        if duplicate_enums:
+            df_errors = pd.DataFrame(
+                [
+                    {
+                        "Enum Path": enum_path,
+                        "Duplicate Values": "; ".join(map(str, duplicates)),
+                    }
+                    for enum_path, duplicates in duplicate_enums.items()
+                ]
+            )
+            error_file_path = f"{self.output_dir}/schema_enum_duplicates.csv"
+            df_errors.to_csv(error_file_path, index=False, encoding="utf-8")
+            relecov_tools.utils.display_dataframe_to_user(
+                name="Schema Enum Duplicates", dataframe=df_errors
+            )
+            stderr.print("[red]Duplicated enum values found. Log saved to:")
+            stderr.print(f"\t{error_file_path}")
+            raise ValueError("Duplicated enum values found in generated schema.")
+
     def get_schema_diff(self, base_schema, new_schema):
         """
         Print the differences between the base schema and the newly generated schema.
@@ -945,7 +1076,7 @@ def save_new_schema(self, json_data):
 
     def _sort_enum_values(self, enum_values: list[str]) -> list[str]:
         """
-        Sort enum values alphabetically while keeping configured special terms last.
+        Deduplicate and sort enum values alphabetically while keeping configured special terms last.
 
         Args:
             enum_values (list[str]): Enum values to sort.
@@ -964,14 +1095,33 @@ def _sort_enum_values(self, enum_values: list[str]) -> list[str]:
             "other": 7,
         }
 
+        unique_values = self._unique_enum_values(enum_values)
+
         def sort_key(value: str):
             normalized_value = value.strip().casefold()
             for last_value, last_order in last_values.items():
                 if last_value in normalized_value:
                     return (1, last_order)
             return (0, normalized_value)
 
-        return sorted(enum_values, key=sort_key)
+        return sorted(unique_values, key=sort_key)
+
+    @staticmethod
+    def _unique_enum_values(enum_values: list) -> list:
+        """Return enum values without duplicates, preserving first occurrence order."""
+        unique_values = []
+        seen = set()
+        for value in enum_values:
+            lookup_value = value.strip() if isinstance(value, str) else value
+            try:
+                is_seen = lookup_value in seen
+            except TypeError:
+                lookup_value = json.dumps(value, sort_keys=True, ensure_ascii=False)
+                is_seen = lookup_value in seen
+            if not is_seen:
+                unique_values.append(value)
+                seen.add(lookup_value)
+        return unique_values
 
     def _parse_enum_values(self, enum_value):
         """
@@ -1092,7 +1242,7 @@ def _load_enum_values_from_sheet(self, enum_value: str) -> list[str]:
             for value in df[column_name].dropna().tolist()
             if str(value).strip()
         ]
-        return list(dict.fromkeys(values))
+        return values
 
     def _load_enum_values_from_file(self, enum_value: str) -> list[str]:
         """
@@ -1286,7 +1436,9 @@ def create_metadatalab_excel(self, json_schema, database_definition=None):
                     )
 
                 def clean_ontologies(enums):
-                    return [re.sub(r"\s*\[.*?\]", "", item).strip() for item in enums]
+                    return self._unique_enum_values(
+                        [re.sub(r"\s*\[.*?\]", "", item).strip() for item in enums]
+                    )
 
                 def resolve_enum_ref(ref: str, enum_defs: dict) -> list[str]:
                     property_key = ref.split("enums/")[-1]

relecov_tools/conf/build_schema_config.json

@@ -2,7 +2,7 @@
     "projects" : {
         "relecov": {
             "host_age_years": {
-                "header_row_idx": 1,
+                "header_row_idx": 5,
                 "max_rows": 1000,
                 "validation_type": "whole",
                 "operator": "between",
@@ -12,7 +12,7 @@
                 "error_title": "Valor no permitido"
             },
             "host_age_months": {
-                "header_row_idx": 1,
+                "header_row_idx": 5,
                 "max_rows": 1000,
                 "validation_type": "whole",
                 "operator": "between",
@@ -22,36 +22,36 @@
                 "error_title": "Valor no permitido"
             },
             "sample_collection_date": {
-                "header_row_idx": 1,
+                "header_row_idx": 5,
                 "max_rows": 1000,
                 "validation_type": "custom",
-                "formula1": "=ISNUMBER({col_letter}5)",
+                "formula1": "=OR({col_letter}5=\"\",IFERROR(AND(ISTEXT({col_letter}5),LEN({col_letter}5)=10,MID({col_letter}5,5,1)=\"-\",MID({col_letter}5,8,1)=\"-\",VALUE(MID({col_letter}5,6,2))>=1,VALUE(MID({col_letter}5,6,2))<=12,VALUE(RIGHT({col_letter}5,2))>=1,VALUE(RIGHT({col_letter}5,2))<=31,ISNUMBER(DATEVALUE({col_letter}5))),FALSE))",
                 "error_message": "Ingrese la fecha en formato correcto YYYY-MM-DD (ejemplo: 2024-02-12).",
                 "error_title": "Formato de fecha incorrecto",
-                "format_cells_as_date": true
+                "format_cells_as_text": true
             },
             "sample_received_date": {
-                "header_row_idx": 1,
+                "header_row_idx": 5,
                 "max_rows": 1000,
                 "validation_type": "custom",
-                "formula1": "=ISNUMBER({col_letter}5)",
+                "formula1": "=OR({col_letter}5=\"\",IFERROR(AND(ISTEXT({col_letter}5),LEN({col_letter}5)=10,MID({col_letter}5,5,1)=\"-\",MID({col_letter}5,8,1)=\"-\",VALUE(MID({col_letter}5,6,2))>=1,VALUE(MID({col_letter}5,6,2))<=12,VALUE(RIGHT({col_letter}5,2))>=1,VALUE(RIGHT({col_letter}5,2))<=31,ISNUMBER(DATEVALUE({col_letter}5))),FALSE))",
                 "error_message": "Ingrese la fecha en formato correcto YYYY-MM-DD (ejemplo: 2024-02-12).",
                 "error_title": "Formato de fecha incorrecto",
-                "format_cells_as_date": true
+                "format_cells_as_text": true
             },
             "sequencing_date": {
-                "header_row_idx": 1,
+                "header_row_idx": 5,
                 "max_rows": 1000,
                 "validation_type": "custom",
-                "formula1": "=ISNUMBER({col_letter}5)",
+                "formula1": "=OR({col_letter}5=\"\",IFERROR(AND(ISTEXT({col_letter}5),LEN({col_letter}5)=10,MID({col_letter}5,5,1)=\"-\",MID({col_letter}5,8,1)=\"-\",VALUE(MID({col_letter}5,6,2))>=1,VALUE(MID({col_letter}5,6,2))<=12,VALUE(RIGHT({col_letter}5,2))>=1,VALUE(RIGHT({col_letter}5,2))<=31,ISNUMBER(DATEVALUE({col_letter}5))),FALSE))",
                 "error_message": "Ingrese la fecha en formato correcto YYYY-MM-DD (ejemplo: 2024-02-12).",
                 "error_title": "Formato de fecha incorrecto",
-                "format_cells_as_date": true
+                "format_cells_as_text": true
             }
         },
         "mepram": {
             "host_age_years": {
-                "header_row_idx": 1,
+                "header_row_idx": 5,
                 "max_rows": 1000,
                 "validation_type": "whole",
                 "operator": "between",
@@ -61,7 +61,7 @@
                 "error_title": "Valor no permitido"
             },
             "host_age_months": {
-                "header_row_idx": 1,
+                "header_row_idx": 5,
                 "max_rows": 1000,
                 "validation_type": "whole",
                 "operator": "between",
@@ -71,22 +71,22 @@
                 "error_title": "Valor no permitido"
             },
             "patient_birth_date": {
-                "header_row_idx": 1,
+                "header_row_idx": 5,
                 "max_rows": 1000,
                 "validation_type": "custom",
-                "formula1": "=ISNUMBER({col_letter}5)",
+                "formula1": "=OR({col_letter}5=\"\",IFERROR(AND(ISTEXT({col_letter}5),LEN({col_letter}5)=10,MID({col_letter}5,5,1)=\"-\",MID({col_letter}5,8,1)=\"-\",VALUE(MID({col_letter}5,6,2))>=1,VALUE(MID({col_letter}5,6,2))<=12,VALUE(RIGHT({col_letter}5,2))>=1,VALUE(RIGHT({col_letter}5,2))<=31,ISNUMBER(DATEVALUE({col_letter}5))),FALSE))",
                 "error_message": "Ingrese la fecha en formato correcto YYYY-MM-DD (ejemplo: 2024-02-12).",
                 "error_title": "Formato de fecha incorrecto",
-                "format_cells_as_date": true
+                "format_cells_as_text": true
             },
             "sample_collection_date": {
-                "header_row_idx": 1,
+                "header_row_idx": 5,
                 "max_rows": 1000,
                 "validation_type": "custom",
-                "formula1": "=ISNUMBER({col_letter}5)",
+                "formula1": "=OR({col_letter}5=\"\",IFERROR(AND(ISTEXT({col_letter}5),LEN({col_letter}5)=10,MID({col_letter}5,5,1)=\"-\",MID({col_letter}5,8,1)=\"-\",VALUE(MID({col_letter}5,6,2))>=1,VALUE(MID({col_letter}5,6,2))<=12,VALUE(RIGHT({col_letter}5,2))>=1,VALUE(RIGHT({col_letter}5,2))<=31,ISNUMBER(DATEVALUE({col_letter}5))),FALSE))",
                 "error_message": "Ingrese la fecha en formato correcto YYYY-MM-DD (ejemplo: 2024-02-12).",
                 "error_title": "Formato de fecha incorrecto",
-                "format_cells_as_date": true
+                "format_cells_as_text": true
             }
         }
     },