diff --git a/argo_metadata_validator/validation.py b/argo_metadata_validator/validation.py index c93322d..a2daf2f 100644 --- a/argo_metadata_validator/validation.py +++ b/argo_metadata_validator/validation.py @@ -13,7 +13,7 @@ from argo_metadata_validator.models.sensor import Sensor from argo_metadata_validator.schema_utils import get_json_validator, infer_schema_from_data, infer_version_from_data from argo_metadata_validator.utils import load_json -from argo_metadata_validator.vocab_utils import expand_vocab, get_all_terms_from_argo_vocabs +from argo_metadata_validator.vocab_utils import VocabTerms, expand_vocab, get_all_terms_from_argo_vocabs def _parse_json_error(error: JsonValidationError) -> ValidationError: @@ -25,11 +25,11 @@ class ArgoValidator: all_json_data: dict[str, Any] = {} # Keyed by the original filename validation_errors: dict[str, list[ValidationError]] = {} # Keyed by the original filename - valid_argo_vocab_terms: list[str] = [] + argo_vocab_terms: VocabTerms def __init__(self): """Initialise by pre-loading the ARGO vocab terms.""" - self.valid_argo_vocab_terms = get_all_terms_from_argo_vocabs() + self.argo_vocab_terms = get_all_terms_from_argo_vocabs() def load_json_data(self, json_files: list[str]): """Take a list of JSON files and load content into memory. @@ -143,6 +143,23 @@ def _validate_vocabs(self, json_data: Any) -> list[ValidationError]: ) return validation_errors + def _is_term_found(self, uri: str, term_list: list[str]): + if uri in term_list: + return True + if re.search(r"_\d+\/$", uri): + # Check if this was a duplicate term (_N added to end) + unduplicate_uri = re.sub(r"_\d+\/$", "/", uri) + return unduplicate_uri in term_list + else: + # No _N at the end so can't be a duplicate term + return False + + def _is_active_term(self, uri: str): + return self._is_term_found(uri, self.argo_vocab_terms.active) + + def _is_deprecated_term(self, uri: str): + return self._is_term_found(uri, self.argo_vocab_terms.deprecated) + def validate_vocab_terms(self, json_data: Any, field: str, sub_fields: list[str]) -> list[ValidationError]: """Check that specific fields in the JSON match ARGO vocab terms. @@ -172,6 +189,10 @@ def validate_vocab_terms(self, json_data: Any, field: str, sub_fields: list[str] # Vocab terms can have optional text enclosed in square brackets val = re.sub(r"\s+\[\w+\]", "", val) val = expand_vocab(context, val) - if val not in self.valid_argo_vocab_terms: - errors.append(ValidationError(message=f"Unknown NSV term: {val}", path=f"{field}.{idx}.{x}")) + if not self._is_active_term(val): + if self._is_deprecated_term(val): + error = ValidationError(message=f"Deprecated NSV term: {val}", path=f"{field}.{idx}.{x}") + else: + error = ValidationError(message=f"Unknown NSV term: {val}", path=f"{field}.{idx}.{x}") + errors.append(error) return errors diff --git a/argo_metadata_validator/vocab_utils.py b/argo_metadata_validator/vocab_utils.py index 7212552..3cd444e 100644 --- a/argo_metadata_validator/vocab_utils.py +++ b/argo_metadata_validator/vocab_utils.py @@ -1,6 +1,7 @@ """Utilities related to NVS/vocabularies.""" import requests +from pydantic import BaseModel NVS_HOST = "http://vocab.nerc.ac.uk" @@ -20,6 +21,13 @@ ] +class VocabTerms(BaseModel): + """Model to hold fetched vocab terms from NVS.""" + + active: list[str] + deprecated: list[str] + + def expand_vocab(context: dict, value: str): """Use context from the JSON to expand vocab terms to full URIs.""" val = value @@ -31,19 +39,21 @@ def expand_vocab(context: dict, value: str): return val -def get_all_terms_from_argo_vocabs() -> list[str]: +def get_all_terms_from_argo_vocabs() -> VocabTerms: """Fetches all active terms from all of the ARGO vocabularies. Returns: list[str]: List of terms as URIs. """ - term_list = [] + terms = VocabTerms(active=[], deprecated=[]) for vocab in ALL_ARGO_VOCABS: - term_list += get_all_terms_from_vocab(vocab) - return term_list + vocab_terms = get_all_terms_from_vocab(vocab) + terms.active += vocab_terms.active + terms.deprecated += vocab_terms.deprecated + return terms -def get_all_terms_from_vocab(vocab: str): +def get_all_terms_from_vocab(vocab: str) -> VocabTerms: """SPARQL query to fetch all active terms from a given vocab. Args: @@ -53,11 +63,10 @@ def get_all_terms_from_vocab(vocab: str): sparql_query = f""" PREFIX skos: PREFIX owl: - SELECT DISTINCT (?c as ?uri) + SELECT DISTINCT (?c as ?uri) ?isDeprecated WHERE {{ <{NVS_HOST}/collection/{vocab}/current/> skos:member ?c . - ?c owl:deprecated ?isDeprecated . - FILTER (?isDeprecated = "false") + ?c owl:deprecated ?isDeprecated }} """ @@ -65,5 +74,10 @@ def get_all_terms_from_vocab(vocab: str): query_url, data=sparql_query, headers={"Content-Type": "application/sparql-query"}, timeout=120 ) resp.raise_for_status() - results = [x["uri"]["value"] for x in resp.json()["results"]["bindings"]] + results = VocabTerms(active=[], deprecated=[]) + for x in resp.json()["results"]["bindings"]: + if x["isDeprecated"]["value"] == "true": + results.deprecated.append(x["uri"]["value"]) + else: + results.active.append(x["uri"]["value"]) return results diff --git a/tests/files/sensor_deprecated_vocab.json b/tests/files/sensor_deprecated_vocab.json new file mode 100644 index 0000000..3953b36 --- /dev/null +++ b/tests/files/sensor_deprecated_vocab.json @@ -0,0 +1,154 @@ +{ + "sensor_info": { + "created_by": "BAK test", + "date_creation": "2023-02-03T07:38:07Z", + "link" : "./argo.sensor.schema.json", + "format_version": "0.4.0", + "contents": "json file to describe a sensor for Argo. v0.4.0 draft", + "sensor_described": "AANDERAA-AANDERAA_OPTODE_4330-3901" + }, + "@context": { + "SDN:R03::": "http://vocab.nerc.ac.uk/collection/R03/current/", + "SDN:R25::": "http://vocab.nerc.ac.uk/collection/R25/current/", + "SDN:R26::": "http://vocab.nerc.ac.uk/collection/R26/current/", + "SDN:R27::": "http://vocab.nerc.ac.uk/collection/R27/current/" + }, + "SENSORS": [ + { + "SENSOR": "SDN:R25::OPTODE_DOXY", + "SENSOR_MAKER": "SDN:R26::AANDERAA", + "SENSOR_MODEL": "SDN:R27::AANDERAA_OPTODE_4330", + "SENSOR_MODEL_FIRMWARE": " ", + "SENSOR_SERIAL_NO": "3901" + } + ], + "PARAMETERS": [ + { + "PARAMETER": "SDN:R03::NB_SAMPLE", + "PARAMETER_SENSOR": "SDN:R25::OPTODE_DOXY", + "PARAMETER_UNITS": "degree", + "PARAMETER_ACCURACY": " ", + "PARAMETER_RESOLUTION": " ", + "PREDEPLOYMENT_CALIB_EQUATION": "none", + "PREDEPLOYMENT_CALIB_COEFFICIENT_LIST": { }, + "PREDEPLOYMENT_CALIB_COMMENT": "Phase measurement with blue excitation light; see TD269 Operating manual oxygen optode 4330, 4835, 483", + "PREDEPLOYMENT_CALIB_DATE": " " + }, + { + "PARAMETER": "SDN:R03::C2PHASE_DOXY", + "PARAMETER_SENSOR": "SDN:R25::OPTODE_DOXY", + "PARAMETER_UNITS": "degree", + "PARAMETER_ACCURACY": " ", + "PARAMETER_RESOLUTION": " ", + "PREDEPLOYMENT_CALIB_EQUATION": "none", + "PREDEPLOYMENT_CALIB_COEFFICIENT_LIST": { }, + "PREDEPLOYMENT_CALIB_COMMENT": "Phase measurement with red excitation light; see TD269 Operating manual oxygen optode 4330, 4835, 4831", + "PREDEPLOYMENT_CALIB_DATE": " " + }, + { + "PARAMETER": "SDN:R03::TEMP_DOXY", + "PARAMETER_SENSOR": "SDN:R25::OPTODE_DOXY", + "PARAMETER_UNITS": "degC", + "PARAMETER_ACCURACY": "0.03", + "PARAMETER_RESOLUTION": "0.01", + "PREDEPLOYMENT_CALIB_EQUATION": "TEMP_DOXY=T0+T1*TEMP_VOLTAGE_DOXY+T2*TEMP_VOLTAGE_DOXY^2+T3*TEMP_VOLTAGE_DOXY^3+T4*TEMP_VOLTAGE_DOXY^4+T5*TEMP_VOLTAGE_DOXY^5; with TEMP_VOLTAGE_DOXY=voltage from thermistor bridge (mV)", + "PREDEPLOYMENT_CALIB_COEFFICIENT_LIST": { + "T0": "not available", + "T1": "not available", + "T2": "not available", + "T3": "not available", + "T4": "not available", + "T5": "not available" + }, + "PREDEPLOYMENT_CALIB_COMMENT": "optode temperature, see TD269 Operating manual oxygen optode 4330, 4835, 4831", + "PREDEPLOYMENT_CALIB_DATE": "2020-01-01T00:00:00Z" + }, + { + "PARAMETER": "SDN:R03::DOXY", + "PARAMETER_SENSOR": "SDN:R25::OPTODE_DOXY", + "PARAMETER_UNITS": "umol/kg", + "PARAMETER_ACCURACY": "8 umol/kg or 10%", + "PARAMETER_RESOLUTION": "1 umol/kg", + "PREDEPLOYMENT_CALIB_EQUATION": "TPHASE_DOXY=C1PHASE_DOXY-C2PHASE_DOXY; Phase_Pcorr=TPHASE_DOXY+Pcoef1*PRES/1000; CalPhase=PhaseCoef0+PhaseCoef1*Phase_Pcorr+PhaseCoef2*Phase_Pcorr^2+PhaseCoef3*Phase_Pcorr^3; MOLAR_DOXY=[((c3+c4*TEMP_DOXY)/(c5+c6*CalPhase))-1]/Ksv; Ksv=c0+c1*TEMP_DOXY+c2*TEMP_DOXY^2; O2=MOLAR_DOXY*Scorr*Pcorr; Scorr=A*exp[PSAL*(B0+B1*Ts+B2*Ts^2+B3*Ts^3)+C0*PSAL^2]; A=[(1013.25-pH2O(TEMP,Spreset))/(1013.25-pH2O(TEMP,PSAL))]; pH2O(TEMP,S)=1013.25*exp[D0+D1*(100/(TEMP+273.15))+D2*ln((TEMP+273.15)/100)+D3*S]; Pcorr=1+((Pcoef2*TEMP+Pcoef3)*PRES)/1000; Ts=ln[(298.15-TEMP)/(273.15+TEMP)]; DOXY=O2/rho, where rho is the potential density [kg/L] calculated from CTD data", + "PREDEPLOYMENT_CALIB_COEFFICIENT_LIST": { + "Spreset": "0", + "Pcoef1": "0.1", + "Pcoef2": "0.00022", + "Pcoef3": "0.0419", + "B0": "0.00624523", + "B1": "-0.00737614", + "B2": "-0.010341", + "B3": "-0.00817083", + "C0": "3.1201642E-3", + "PhaseCoef0": "-1.652", + "PhaseCoef1": "1", + "PhaseCoef2": "0", + "PhaseCoef3": "0", + "c0": "0.00261275", + "c1": "0.00011268", + "c2": "2.2309e-06", + "c3": "200.183", + "c4": "-0.223497", + "c5": "-43.6776", + "c6": "4.10578", + "D0": "24.4543", + "D1": "-67.4509", + "D2": "-4.8489", + "D3": "-0.000544", + "C1": "-285.56594E-6", + "C2": "316.32993E-9", + "C3": "-1.0767272E-6" + }, + "PREDEPLOYMENT_CALIB_COMMENT": "see TD269 Operating manual oxygen optode 4330, 4835, 4831; see Processing Argo OXYGEN data at the DAC level, Version 2.2 (DOI: http://dx.doi.org/10.13155/39795)", + "PREDEPLOYMENT_CALIB_DATE": "2020-01-01T00:00:00Z", + "parameter_vendorinfo": { + "example": "Vendors can add their own parameter info here; they can add any field they like as a fieldname below parameter_vendorinfo" + }, + "predeployment_vendorinfo": { + "example": "Vendors can add their own predeployment info here; they can add any field they like as a fieldname below predeployment_vendorinfo" + } + }, + { + "PARAMETER": "SDN:R03::PPOX_DOXY", + "PARAMETER_SENSOR": "SDN:R25::OPTODE_DOXY", + "PARAMETER_UNITS": "mbar", + "PARAMETER_ACCURACY": " ", + "PARAMETER_RESOLUTION": " ", + "PREDEPLOYMENT_CALIB_EQUATION": "TPHASE_DOXY=C1PHASE_DOXY-C2PHASE_DOXY; Phase_Pcorr=TPHASE_DOXY+Pcoef1*PRES/1000; CalPhase=PhaseCoef0+PhaseCoef1*Phase_Pcorr+PhaseCoef2*Phase_Pcorr^2+PhaseCoef3*Phase_Pcorr^3; Ksv=c0+c1*TEMP_DOXY+c2*TEMP_DOXY^2; MOLAR_DOXY=[((c3+c4*TEMP_DOXY)/(c5+c6*CalPhase))-1]/Ksv; Pcorr=1+((Pcoef2*TEMP+Pcoef3)*PRES)/1000; MOLAR_DOXY=MOLAR_DOXY*Pcorr; pH2Osat=1013.25*exp[D0+D1*(100/(TEMP+273.15))+D2*ln((TEMP+273.15)/100)]; Tcorr=44.6596*exp[2.00907+3.22014*Ts+4.05010*Ts^2+4.94457*Ts^3-2.56847e-1*Ts^4+3.88767*Ts^5]; Ts=ln[(298.15-TEMP)/(273.15+TEMP)]; PPOX_DOXY=MOLAR_DOXY*(0.20946*(1013.25-pH2Osat))/Tcorr*exp[0.317*PRES/(8.314*(TEMP+273.15))]", + "PREDEPLOYMENT_CALIB_COEFFICIENT_LIST": { + "Pcoef1": "0.1", + "Pcoef2": "0.00022", + "Pcoef3": "0.0419", + "PhaseCoef0": "-1.652", + "PhaseCoef1": "1", + "PhaseCoef2": "0", + "PhaseCoef3": "0", + "c0": "0.00261275", + "c1": "0.00011268", + "c2": "2.2309e-06", + "c3": "200.183", + "c4": "-0.223497", + "c5": "-43.6776", + "c6": "4.10578", + "D0": "24.4543", + "D1": "-67.4509", + "D2": "-4.8489", + "C0": "3.1201642E-3", + "C1": "-285.56594E-6", + "C2": "316.32993E-9", + "C3": "-1.0767272E-6" + }, + "PREDEPLOYMENT_CALIB_COMMENT": "see TD269 Operating manual oxygen optode 4330, 4835, 4831; see Processing Argo OXYGEN data at the DAC level, Version 2.2 (DOI: http://dx.doi.org/10.13155/39795)", + "PREDEPLOYMENT_CALIB_DATE": "2020-01-01T00:00:00Z", + "parameter_vendorinfo": { + "example": "Vendors can add their own parameter info here; they can add any field they like as a fieldname below parameter_vendorinfo" + }, + "predeployment_vendorinfo": { + "example": "Vendors can add their own predeployment info here; they can add any field they like as a fieldname below predeployment_vendorinfo" + } + } + ], + "instrument_vendorinfo": { + "example": "Vendors can add something about the whole instrument here; they can add field they like as a fieldname below instrument_vendorinfo" + } + } \ No newline at end of file diff --git a/tests/files/valid_float.json b/tests/files/valid_float.json index bb0ae56..611c132 100644 --- a/tests/files/valid_float.json +++ b/tests/files/valid_float.json @@ -174,6 +174,13 @@ "SENSOR_MODEL_FIRMWARE": " ", "SENSOR_SERIAL_NO": "3901" }, + { + "SENSOR": "SDN:R25::OPTODE_DOXY_2", + "SENSOR_MAKER": "SDN:R26::AANDERAA", + "SENSOR_MODEL": "SDN:R27::AANDERAA_OPTODE_4330", + "SENSOR_MODEL_FIRMWARE": " ", + "SENSOR_SERIAL_NO": "3901" + }, { "SENSOR": "SDN:R25::CTD_PRES", "SENSOR_MAKER": "SDN:R26::RBR", @@ -496,6 +503,17 @@ "PREDEPLOYMENT_CALIB_COMMENT": "Phase measurement with blue excitation light; see TD269 Operating manual oxygen optode 4330, 4835, 483", "PREDEPLOYMENT_CALIB_DATE": " " }, + { + "PARAMETER": "SDN:R03::C1PHASE_DOXY_2", + "PARAMETER_SENSOR": "SDN:R25::OPTODE_DOXY_2", + "PARAMETER_UNITS": "degree", + "PARAMETER_ACCURACY": " ", + "PARAMETER_RESOLUTION": " ", + "PREDEPLOYMENT_CALIB_EQUATION": "none", + "PREDEPLOYMENT_CALIB_COEFFICIENT_LIST": {}, + "PREDEPLOYMENT_CALIB_COMMENT": "Phase measurement with blue excitation light; see TD269 Operating manual oxygen optode 4330, 4835, 483", + "PREDEPLOYMENT_CALIB_DATE": " " + }, { "PARAMETER": "SDN:R03::C2PHASE_DOXY", "PARAMETER_SENSOR": "SDN:R25::OPTODE_DOXY", diff --git a/tests/integration_tests/test_file_validation.py b/tests/integration_tests/test_file_validation.py index 6cb6c46..d0e8a88 100644 --- a/tests/integration_tests/test_file_validation.py +++ b/tests/integration_tests/test_file_validation.py @@ -43,6 +43,17 @@ ] }, ], + [ + "sensor_deprecated_vocab.json", + { + "sensor_deprecated_vocab.json": [ + ValidationError( + message="Deprecated NSV term: http://vocab.nerc.ac.uk/collection/R03/current/NB_SAMPLE/", + path="PARAMETERS.0.PARAMETER", + ) + ] + }, + ], ], ) def test_validating_files(file_path, expected_output): diff --git a/tests/unit_tests/test_vocab_utils.py b/tests/unit_tests/test_vocab_utils.py index 46fd85c..3b1049e 100644 --- a/tests/unit_tests/test_vocab_utils.py +++ b/tests/unit_tests/test_vocab_utils.py @@ -6,6 +6,7 @@ from argo_metadata_validator.vocab_utils import ( ALL_ARGO_VOCABS, NVS_HOST, + VocabTerms, expand_vocab, get_all_terms_from_argo_vocabs, get_all_terms_from_vocab, @@ -34,15 +35,18 @@ def test_expand_vocab(input_val, expected_result): def test_get_all_terms_from_argo_vocabs(mocker): """Test for get_all_terms_from_argo_vocabs calling mocked version of sub-method.""" - mock_get = mocker.patch("argo_metadata_validator.vocab_utils.get_all_terms_from_vocab", return_value=["1"]) + mock_get = mocker.patch( + "argo_metadata_validator.vocab_utils.get_all_terms_from_vocab", + return_value=VocabTerms(active=["1"], deprecated=[]), + ) result = get_all_terms_from_argo_vocabs() # Check the per-vocab call happens the right number of times assert mock_get.call_count == len(ALL_ARGO_VOCABS) # Check that result is correctly a list of strings - assert isinstance(result, list) - assert all(isinstance(x, str) for x in result) + assert isinstance(result.active, list) + assert all(isinstance(x, str) for x in result.active) def test_get_all_terms_from_vocab(): @@ -50,8 +54,8 @@ def test_get_all_terms_from_vocab(): example_response = { "results": { "bindings": [ - {"uri": {"value": "http://vocab/hi"}}, - {"uri": {"value": "http://vocab/bye"}}, + {"uri": {"value": "http://vocab/hi"}, "isDeprecated": {"value": "false"}}, + {"uri": {"value": "http://vocab/bye"}, "isDeprecated": {"value": "true"}}, ] } } @@ -60,4 +64,5 @@ def test_get_all_terms_from_vocab(): mock_req.post(f"{NVS_HOST}/sparql/sparql", json=example_response) result = get_all_terms_from_vocab("R01") - assert result == ["http://vocab/hi", "http://vocab/bye"] + assert result.active == ["http://vocab/hi"] + assert result.deprecated == ["http://vocab/bye"]