Feature/validation fixes

argo_metadata_validator/validation.py

@@ -13,7 +13,7 @@
 from argo_metadata_validator.models.sensor import Sensor
 from argo_metadata_validator.schema_utils import get_json_validator, infer_schema_from_data, infer_version_from_data
 from argo_metadata_validator.utils import load_json
-from argo_metadata_validator.vocab_utils import expand_vocab, get_all_terms_from_argo_vocabs
+from argo_metadata_validator.vocab_utils import VocabTerms, expand_vocab, get_all_terms_from_argo_vocabs
 
 
 def _parse_json_error(error: JsonValidationError) -> ValidationError:
@@ -25,11 +25,11 @@ class ArgoValidator:
 
     all_json_data: dict[str, Any] = {}  # Keyed by the original filename
     validation_errors: dict[str, list[ValidationError]] = {}  # Keyed by the original filename
-    valid_argo_vocab_terms: list[str] = []
+    argo_vocab_terms: VocabTerms
 
     def __init__(self):
         """Initialise by pre-loading the ARGO vocab terms."""
-        self.valid_argo_vocab_terms = get_all_terms_from_argo_vocabs()
+        self.argo_vocab_terms = get_all_terms_from_argo_vocabs()
 
     def load_json_data(self, json_files: list[str]):
         """Take a list of JSON files and load content into memory.
@@ -143,6 +143,23 @@ def _validate_vocabs(self, json_data: Any) -> list[ValidationError]:
             )
         return validation_errors
 
+    def _is_term_found(self, uri: str, term_list: list[str]):
+        if uri in term_list:
+            return True
+        if re.search(r"_\d+\/$", uri):
+            # Check if this was a duplicate term (_N added to end)
+            unduplicate_uri = re.sub(r"_\d+\/$", "/", uri)
+            return unduplicate_uri in term_list
+        else:
+            # No _N at the end so can't be a duplicate term
+            return False
+
+    def _is_active_term(self, uri: str):
+        return self._is_term_found(uri, self.argo_vocab_terms.active)
+
+    def _is_deprecated_term(self, uri: str):
+        return self._is_term_found(uri, self.argo_vocab_terms.deprecated)
+
     def validate_vocab_terms(self, json_data: Any, field: str, sub_fields: list[str]) -> list[ValidationError]:
         """Check that specific fields in the JSON match ARGO vocab terms.
 
@@ -172,6 +189,10 @@ def validate_vocab_terms(self, json_data: Any, field: str, sub_fields: list[str]
                     # Vocab terms can have optional text enclosed in square brackets
                     val = re.sub(r"\s+\[\w+\]", "", val)
                     val = expand_vocab(context, val)
-                    if val not in self.valid_argo_vocab_terms:
-                        errors.append(ValidationError(message=f"Unknown NSV term: {val}", path=f"{field}.{idx}.{x}"))
+                    if not self._is_active_term(val):
+                        if self._is_deprecated_term(val):
+                            error = ValidationError(message=f"Deprecated NSV term: {val}", path=f"{field}.{idx}.{x}")
+                        else:
+                            error = ValidationError(message=f"Unknown NSV term: {val}", path=f"{field}.{idx}.{x}")
+                        errors.append(error)
         return errors

argo_metadata_validator/vocab_utils.py

@@ -1,6 +1,7 @@
 """Utilities related to NVS/vocabularies."""
 
 import requests
+from pydantic import BaseModel
 
 NVS_HOST = "http://vocab.nerc.ac.uk"
 
@@ -20,6 +21,13 @@
 ]
 
 
+class VocabTerms(BaseModel):
+    """Model to hold fetched vocab terms from NVS."""
+
+    active: list[str]
+    deprecated: list[str]
+
+
 def expand_vocab(context: dict, value: str):
     """Use context from the JSON to expand vocab terms to full URIs."""
     val = value
@@ -31,19 +39,21 @@ def expand_vocab(context: dict, value: str):
     return val
 
 
-def get_all_terms_from_argo_vocabs() -> list[str]:
+def get_all_terms_from_argo_vocabs() -> VocabTerms:
     """Fetches all active terms from all of the ARGO vocabularies.
 
     Returns:
         list[str]: List of terms as URIs.
     """
-    term_list = []
+    terms = VocabTerms(active=[], deprecated=[])
     for vocab in ALL_ARGO_VOCABS:
-        term_list += get_all_terms_from_vocab(vocab)
-    return term_list
+        vocab_terms = get_all_terms_from_vocab(vocab)
+        terms.active += vocab_terms.active
+        terms.deprecated += vocab_terms.deprecated
+    return terms
 
 
-def get_all_terms_from_vocab(vocab: str):
+def get_all_terms_from_vocab(vocab: str) -> VocabTerms:
     """SPARQL query to fetch all active terms from a given vocab.
 
     Args:
@@ -53,17 +63,21 @@ def get_all_terms_from_vocab(vocab: str):
     sparql_query = f"""
     PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
     PREFIX owl: <http://www.w3.org/2002/07/owl#>
-    SELECT DISTINCT (?c as ?uri)
+    SELECT DISTINCT (?c as ?uri) ?isDeprecated
     WHERE {{
         <{NVS_HOST}/collection/{vocab}/current/> skos:member ?c .
-        ?c owl:deprecated ?isDeprecated .
-        FILTER (?isDeprecated = "false")
+        ?c owl:deprecated ?isDeprecated
     }}
     """
 
     resp = requests.post(
         query_url, data=sparql_query, headers={"Content-Type": "application/sparql-query"}, timeout=120
     )
     resp.raise_for_status()
-    results = [x["uri"]["value"] for x in resp.json()["results"]["bindings"]]
+    results = VocabTerms(active=[], deprecated=[])
+    for x in resp.json()["results"]["bindings"]:
+        if x["isDeprecated"]["value"] == "true":
+            results.deprecated.append(x["uri"]["value"])
+        else:
+            results.active.append(x["uri"]["value"])
     return results

tests/files/sensor_deprecated_vocab.json

@@ -0,0 +1,154 @@
+{
+    "sensor_info": {
+      "created_by": "BAK test",
+      "date_creation": "2023-02-03T07:38:07Z",
+      "link" : "./argo.sensor.schema.json",
+      "format_version": "0.4.0",
+      "contents": "json file to describe a sensor for Argo. v0.4.0 draft",
+      "sensor_described": "AANDERAA-AANDERAA_OPTODE_4330-3901"
+    },
+    "@context": {
+      "SDN:R03::": "http://vocab.nerc.ac.uk/collection/R03/current/",
+      "SDN:R25::": "http://vocab.nerc.ac.uk/collection/R25/current/",
+      "SDN:R26::": "http://vocab.nerc.ac.uk/collection/R26/current/",
+      "SDN:R27::": "http://vocab.nerc.ac.uk/collection/R27/current/"
+    },
+  "SENSORS": [
+      {
+        "SENSOR": "SDN:R25::OPTODE_DOXY",
+        "SENSOR_MAKER": "SDN:R26::AANDERAA",
+        "SENSOR_MODEL": "SDN:R27::AANDERAA_OPTODE_4330",
+        "SENSOR_MODEL_FIRMWARE": " ",
+        "SENSOR_SERIAL_NO": "3901"
+      }
+    ],
+    "PARAMETERS": [
+      {
+        "PARAMETER": "SDN:R03::NB_SAMPLE",
+        "PARAMETER_SENSOR": "SDN:R25::OPTODE_DOXY",
+        "PARAMETER_UNITS": "degree",
+        "PARAMETER_ACCURACY": " ",
+        "PARAMETER_RESOLUTION": " ",
+        "PREDEPLOYMENT_CALIB_EQUATION": "none",
+        "PREDEPLOYMENT_CALIB_COEFFICIENT_LIST": {  },
+        "PREDEPLOYMENT_CALIB_COMMENT": "Phase measurement with blue excitation light; see TD269 Operating manual oxygen optode 4330, 4835, 483",
+        "PREDEPLOYMENT_CALIB_DATE": " "
+      },
+      {
+        "PARAMETER": "SDN:R03::C2PHASE_DOXY",
+        "PARAMETER_SENSOR": "SDN:R25::OPTODE_DOXY",
+        "PARAMETER_UNITS": "degree",
+        "PARAMETER_ACCURACY": " ",
+        "PARAMETER_RESOLUTION": " ",
+        "PREDEPLOYMENT_CALIB_EQUATION": "none",
+        "PREDEPLOYMENT_CALIB_COEFFICIENT_LIST": { },
+        "PREDEPLOYMENT_CALIB_COMMENT": "Phase measurement with red excitation light; see TD269 Operating manual oxygen optode 4330, 4835, 4831",
+        "PREDEPLOYMENT_CALIB_DATE": " "    
+      },
+      {
+        "PARAMETER": "SDN:R03::TEMP_DOXY",
+        "PARAMETER_SENSOR": "SDN:R25::OPTODE_DOXY",
+        "PARAMETER_UNITS": "degC",
+        "PARAMETER_ACCURACY": "0.03",
+        "PARAMETER_RESOLUTION": "0.01",
+        "PREDEPLOYMENT_CALIB_EQUATION": "TEMP_DOXY=T0+T1*TEMP_VOLTAGE_DOXY+T2*TEMP_VOLTAGE_DOXY^2+T3*TEMP_VOLTAGE_DOXY^3+T4*TEMP_VOLTAGE_DOXY^4+T5*TEMP_VOLTAGE_DOXY^5; with TEMP_VOLTAGE_DOXY=voltage from thermistor bridge (mV)",
+        "PREDEPLOYMENT_CALIB_COEFFICIENT_LIST": {
+          "T0": "not available",
+          "T1": "not available",
+          "T2": "not available",
+          "T3": "not available",
+          "T4": "not available",
+          "T5": "not available"
+        },
+        "PREDEPLOYMENT_CALIB_COMMENT": "optode temperature, see TD269 Operating manual oxygen optode 4330, 4835, 4831",
+        "PREDEPLOYMENT_CALIB_DATE": "2020-01-01T00:00:00Z"
+      },
+      {
+        "PARAMETER": "SDN:R03::DOXY",
+        "PARAMETER_SENSOR": "SDN:R25::OPTODE_DOXY",
+        "PARAMETER_UNITS": "umol/kg",
+        "PARAMETER_ACCURACY": "8 umol/kg or 10%",
+        "PARAMETER_RESOLUTION": "1 umol/kg",
+        "PREDEPLOYMENT_CALIB_EQUATION": "TPHASE_DOXY=C1PHASE_DOXY-C2PHASE_DOXY; Phase_Pcorr=TPHASE_DOXY+Pcoef1*PRES/1000; CalPhase=PhaseCoef0+PhaseCoef1*Phase_Pcorr+PhaseCoef2*Phase_Pcorr^2+PhaseCoef3*Phase_Pcorr^3; MOLAR_DOXY=[((c3+c4*TEMP_DOXY)/(c5+c6*CalPhase))-1]/Ksv; Ksv=c0+c1*TEMP_DOXY+c2*TEMP_DOXY^2; O2=MOLAR_DOXY*Scorr*Pcorr; Scorr=A*exp[PSAL*(B0+B1*Ts+B2*Ts^2+B3*Ts^3)+C0*PSAL^2]; A=[(1013.25-pH2O(TEMP,Spreset))/(1013.25-pH2O(TEMP,PSAL))]; pH2O(TEMP,S)=1013.25*exp[D0+D1*(100/(TEMP+273.15))+D2*ln((TEMP+273.15)/100)+D3*S]; Pcorr=1+((Pcoef2*TEMP+Pcoef3)*PRES)/1000; Ts=ln[(298.15-TEMP)/(273.15+TEMP)]; DOXY=O2/rho, where rho is the potential density [kg/L] calculated from CTD data",
+        "PREDEPLOYMENT_CALIB_COEFFICIENT_LIST": {
+          "Spreset": "0",
+          "Pcoef1": "0.1",
+          "Pcoef2": "0.00022",
+          "Pcoef3": "0.0419",
+          "B0": "0.00624523",
+          "B1": "-0.00737614",
+          "B2": "-0.010341",
+          "B3": "-0.00817083",
+          "C0": "3.1201642E-3",
+          "PhaseCoef0": "-1.652",
+          "PhaseCoef1": "1",
+          "PhaseCoef2": "0",
+          "PhaseCoef3": "0",
+          "c0": "0.00261275",
+          "c1": "0.00011268",
+          "c2": "2.2309e-06",
+          "c3": "200.183",
+          "c4": "-0.223497",
+          "c5": "-43.6776",
+          "c6": "4.10578",
+          "D0": "24.4543",
+          "D1": "-67.4509",
+          "D2": "-4.8489",
+          "D3": "-0.000544",
+          "C1": "-285.56594E-6",
+          "C2": "316.32993E-9",
+          "C3": "-1.0767272E-6"
+        },
+        "PREDEPLOYMENT_CALIB_COMMENT": "see TD269 Operating manual oxygen optode 4330, 4835, 4831; see Processing Argo OXYGEN data at the DAC level, Version 2.2 (DOI: http://dx.doi.org/10.13155/39795)",
+        "PREDEPLOYMENT_CALIB_DATE": "2020-01-01T00:00:00Z",
+        "parameter_vendorinfo": {
+          "example": "Vendors can add their own parameter info here; they can add any field they like as a fieldname below parameter_vendorinfo"
+        },
+        "predeployment_vendorinfo": {
+          "example": "Vendors can add their own predeployment info here; they can add any field they like as a fieldname below predeployment_vendorinfo"
+        }
+      },
+      {
+        "PARAMETER": "SDN:R03::PPOX_DOXY",
+        "PARAMETER_SENSOR": "SDN:R25::OPTODE_DOXY",
+        "PARAMETER_UNITS": "mbar",
+        "PARAMETER_ACCURACY": " ",
+        "PARAMETER_RESOLUTION": " ",
+        "PREDEPLOYMENT_CALIB_EQUATION": "TPHASE_DOXY=C1PHASE_DOXY-C2PHASE_DOXY; Phase_Pcorr=TPHASE_DOXY+Pcoef1*PRES/1000; CalPhase=PhaseCoef0+PhaseCoef1*Phase_Pcorr+PhaseCoef2*Phase_Pcorr^2+PhaseCoef3*Phase_Pcorr^3; Ksv=c0+c1*TEMP_DOXY+c2*TEMP_DOXY^2; MOLAR_DOXY=[((c3+c4*TEMP_DOXY)/(c5+c6*CalPhase))-1]/Ksv; Pcorr=1+((Pcoef2*TEMP+Pcoef3)*PRES)/1000; MOLAR_DOXY=MOLAR_DOXY*Pcorr; pH2Osat=1013.25*exp[D0+D1*(100/(TEMP+273.15))+D2*ln((TEMP+273.15)/100)]; Tcorr=44.6596*exp[2.00907+3.22014*Ts+4.05010*Ts^2+4.94457*Ts^3-2.56847e-1*Ts^4+3.88767*Ts^5]; Ts=ln[(298.15-TEMP)/(273.15+TEMP)]; PPOX_DOXY=MOLAR_DOXY*(0.20946*(1013.25-pH2Osat))/Tcorr*exp[0.317*PRES/(8.314*(TEMP+273.15))]",
+        "PREDEPLOYMENT_CALIB_COEFFICIENT_LIST": {
+          "Pcoef1": "0.1",
+          "Pcoef2": "0.00022",
+          "Pcoef3": "0.0419",
+          "PhaseCoef0": "-1.652",
+          "PhaseCoef1": "1",
+          "PhaseCoef2": "0",
+          "PhaseCoef3": "0",
+          "c0": "0.00261275",
+          "c1": "0.00011268",
+          "c2": "2.2309e-06",
+          "c3": "200.183",
+          "c4": "-0.223497",
+          "c5": "-43.6776",
+          "c6": "4.10578",
+          "D0": "24.4543",
+          "D1": "-67.4509",
+          "D2": "-4.8489",
+          "C0": "3.1201642E-3",
+          "C1": "-285.56594E-6",
+          "C2": "316.32993E-9",
+          "C3": "-1.0767272E-6"
+        },
+        "PREDEPLOYMENT_CALIB_COMMENT": "see TD269 Operating manual oxygen optode 4330, 4835, 4831; see Processing Argo OXYGEN data at the DAC level, Version 2.2 (DOI: http://dx.doi.org/10.13155/39795)",
+        "PREDEPLOYMENT_CALIB_DATE": "2020-01-01T00:00:00Z",
+        "parameter_vendorinfo": {
+          "example": "Vendors can add their own parameter info here; they can add any field they like as a fieldname below parameter_vendorinfo"
+        },
+        "predeployment_vendorinfo": {
+          "example": "Vendors can add their own predeployment info here; they can add any field they like as a fieldname below predeployment_vendorinfo"
+        }
+      }
+    ],
+    "instrument_vendorinfo": {
+      "example": "Vendors can add something about the whole instrument here; they can add field they like as a fieldname below instrument_vendorinfo"
+    }
+  }

tests/files/valid_float.json

@@ -174,6 +174,13 @@
       "SENSOR_MODEL_FIRMWARE": " ",
       "SENSOR_SERIAL_NO": "3901"
     },
+    {
+      "SENSOR": "SDN:R25::OPTODE_DOXY_2",
+      "SENSOR_MAKER": "SDN:R26::AANDERAA",
+      "SENSOR_MODEL": "SDN:R27::AANDERAA_OPTODE_4330",
+      "SENSOR_MODEL_FIRMWARE": " ",
+      "SENSOR_SERIAL_NO": "3901"
+    },
     {
       "SENSOR": "SDN:R25::CTD_PRES",
       "SENSOR_MAKER": "SDN:R26::RBR",
@@ -496,6 +503,17 @@
       "PREDEPLOYMENT_CALIB_COMMENT": "Phase measurement with blue excitation light; see TD269 Operating manual oxygen optode 4330, 4835, 483",
       "PREDEPLOYMENT_CALIB_DATE": " "
     },
+    {
+      "PARAMETER": "SDN:R03::C1PHASE_DOXY_2",
+      "PARAMETER_SENSOR": "SDN:R25::OPTODE_DOXY_2",
+      "PARAMETER_UNITS": "degree",
+      "PARAMETER_ACCURACY": " ",
+      "PARAMETER_RESOLUTION": " ",
+      "PREDEPLOYMENT_CALIB_EQUATION": "none",
+      "PREDEPLOYMENT_CALIB_COEFFICIENT_LIST": {},
+      "PREDEPLOYMENT_CALIB_COMMENT": "Phase measurement with blue excitation light; see TD269 Operating manual oxygen optode 4330, 4835, 483",
+      "PREDEPLOYMENT_CALIB_DATE": " "
+    },
     {
       "PARAMETER": "SDN:R03::C2PHASE_DOXY",
       "PARAMETER_SENSOR": "SDN:R25::OPTODE_DOXY",

tests/integration_tests/test_file_validation.py

@@ -43,6 +43,17 @@
                 ]
             },
         ],
+        [
+            "sensor_deprecated_vocab.json",
+            {
+                "sensor_deprecated_vocab.json": [
+                    ValidationError(
+                        message="Deprecated NSV term: http://vocab.nerc.ac.uk/collection/R03/current/NB_SAMPLE/",
+                        path="PARAMETERS.0.PARAMETER",
+                    )
+                ]
+            },
+        ],
     ],
 )
 def test_validating_files(file_path, expected_output):

tests/unit_tests/test_vocab_utils.py

@@ -6,6 +6,7 @@
 from argo_metadata_validator.vocab_utils import (
     ALL_ARGO_VOCABS,
     NVS_HOST,
+    VocabTerms,
     expand_vocab,
     get_all_terms_from_argo_vocabs,
     get_all_terms_from_vocab,
@@ -34,24 +35,27 @@ def test_expand_vocab(input_val, expected_result):
 
 def test_get_all_terms_from_argo_vocabs(mocker):
     """Test for get_all_terms_from_argo_vocabs calling mocked version of sub-method."""
-    mock_get = mocker.patch("argo_metadata_validator.vocab_utils.get_all_terms_from_vocab", return_value=["1"])
+    mock_get = mocker.patch(
+        "argo_metadata_validator.vocab_utils.get_all_terms_from_vocab",
+        return_value=VocabTerms(active=["1"], deprecated=[]),
+    )
 
     result = get_all_terms_from_argo_vocabs()
 
     # Check the per-vocab call happens the right number of times
     assert mock_get.call_count == len(ALL_ARGO_VOCABS)
     # Check that result is correctly a list of strings
-    assert isinstance(result, list)
-    assert all(isinstance(x, str) for x in result)
+    assert isinstance(result.active, list)
+    assert all(isinstance(x, str) for x in result.active)
 
 
 def test_get_all_terms_from_vocab():
     """Simple unit test for get_all_terms_from_vocab, mocking the HTTP call."""
     example_response = {
         "results": {
             "bindings": [
-                {"uri": {"value": "http://vocab/hi"}},
-                {"uri": {"value": "http://vocab/bye"}},
+                {"uri": {"value": "http://vocab/hi"}, "isDeprecated": {"value": "false"}},
+                {"uri": {"value": "http://vocab/bye"}, "isDeprecated": {"value": "true"}},
             ]
         }
     }
@@ -60,4 +64,5 @@ def test_get_all_terms_from_vocab():
         mock_req.post(f"{NVS_HOST}/sparql/sparql", json=example_response)
         result = get_all_terms_from_vocab("R01")
 
-    assert result == ["http://vocab/hi", "http://vocab/bye"]
+    assert result.active == ["http://vocab/hi"]
+    assert result.deprecated == ["http://vocab/bye"]