diff --git a/CHANGES.rst b/CHANGES.rst index e49c9a0bdd..57692ebf3b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,6 +4,7 @@ Changelog 2.6.0 (unreleased) ------------------ +- #2682 Added Limit of Quantification (LOQ) for services and analyses - #2687 Remove legacy and obsolete rejection.js - #2685 Fix missing default instrument import template - #2684 Fix Traceback for unicode titled instruments diff --git a/src/bika/lims/content/abstractanalysis.py b/src/bika/lims/content/abstractanalysis.py index 292680cf32..90e7414a49 100644 --- a/src/bika/lims/content/abstractanalysis.py +++ b/src/bika/lims/content/abstractanalysis.py @@ -52,6 +52,7 @@ from Products.Archetypes.Schema import Schema from Products.CMFCore.permissions import View from senaite.core.browser.fields.datetime import DateTimeField +from senaite.core.i18n import translate as t from senaite.core.permissions import FieldEditAnalysisResult from senaite.core.permissions import ViewResults from six import string_types @@ -227,66 +228,67 @@ def getVerificators(self): return verifiers @security.public - def getDefaultUncertainty(self, result=None): + def getDefaultUncertainty(self): """Return the uncertainty value, if the result falls within specified ranges for the service from which this analysis was derived. """ - - if result is None: - result = self.getResult() + result = self.getResult() + if not api.is_floatable(result): + return None uncertainties = self.getUncertainties() - if uncertainties: - try: - res = float(result) - except (TypeError, ValueError): - # if analysis result is not a number, then we assume in range - return None - - for d in uncertainties: - - # convert to min/max - unc_min = api.to_float(d["intercept_min"], default=0) - unc_max = api.to_float(d["intercept_max"], default=0) - - if unc_min <= res and res <= unc_max: - _err = str(d["errorvalue"]).strip() - if _err.endswith("%"): - try: - percvalue = float(_err.replace("%", "")) - except ValueError: - return None - # calculate uncertainty from result - uncertainty = res / 100 * percvalue - else: - uncertainty = api.to_float(_err, default=0) + if not uncertainties: + return None + + result = api.to_float(result) + for record in uncertainties: + + # convert to min/max + unc_min = api.to_float(record["intercept_min"], default=0) + unc_max = api.to_float(record["intercept_max"], default=0) + + if unc_min <= result <= unc_max: + # result is within the range defined for this uncertainty + uncertainty = str(record["errorvalue"]).strip() + if uncertainty.endswith("%"): + # uncertainty expressed as a percentage of the result + try: + percentage = float(uncertainty.replace("%", "")) + uncertainty = result / 100 * percentage + except ValueError: + return None + else: + uncertainty = api.to_float(uncertainty, default=0) + + # convert back to string value + return api.float_to_string(uncertainty, default=None) - # convert back to string value - return api.float_to_string(uncertainty, default=None) return None @security.public - def getUncertainty(self, result=None): - """Returns the uncertainty for this analysis and result. + def getUncertainty(self): + """Returns the uncertainty for this analysis. Returns the value from Schema's Uncertainty field if the Service has the option 'Allow manual uncertainty'. Otherwise, do a callback to getDefaultUncertainty(). - Returns empty string if no result specified and the current result for this - analysis is below or above detections limits. + Returns None if no result specified and the current result for this + analysis is outside of the quantifiable range. """ - uncertainty = self.getField("Uncertainty").get(self) - if result is None: - if self.isAboveUpperDetectionLimit(): - return None - if self.isBelowLowerDetectionLimit(): - return None + if self.isOutsideTheQuantifiableRange(): + # does not make sense to display uncertainty if the result is + # outside of the quantifiable because the measurement is not + # reliable or accurate enough to confidently quantify the analyte + return None + uncertainty = self.getField("Uncertainty").get(self) if uncertainty and self.getAllowManualUncertainty(): + # the uncertainty has been manually set on results introduction return api.float_to_string(uncertainty, default=None) - return self.getDefaultUncertainty(result) + # fallback to the default uncertainty for this analysis + return self.getDefaultUncertainty() @security.public def setUncertainty(self, unc): @@ -295,11 +297,7 @@ def setUncertainty(self, unc): If the result is a Detection Limit or the value is below LDL or upper UDL, set the uncertainty to None`` """ - # Uncertainty calculation on DL - # https://jira.bikalabs.com/browse/LIMS-1808 - if self.isAboveUpperDetectionLimit(): - unc = None - if self.isBelowLowerDetectionLimit(): + if self.isOutsideTheQuantifiableRange(): unc = None field = self.getField("Uncertainty") @@ -414,6 +412,62 @@ def isAboveUpperDetectionLimit(self): return False + @security.public + def getLowerLimitOfQuantification(self): + """Returns the Lower Limit of Quantification (LLOQ) for the current + analysis. If the defined LLOQ is lower than the Lower Limit of + Detection (LLOD), the function returns the LLOD instead. This ensures + the result respects the detection threshold + """ + llod = self.getLowerDetectionLimit() + lloq = self.getField("LowerLimitOfQuantification").get(self) + return llod if api.to_float(lloq) < api.to_float(llod) else lloq + + @security.public + def getUpperLimitOfQuantification(self): + """Returns the Upper Limit of Quantification (ULOQ) for the current + analysis. If the defined ULOQ is greater than the Upper Limit of + Detection (ULOD), the function returns the ULOD instead. This ensures + the result respects the detection threshold + """ + ulod = self.getUpperDetectionLimit() + uloq = self.getField("UpperLimitOfQuantification").get(self) + return ulod if api.to_float(uloq) > api.to_float(ulod) else uloq + + @security.public + def isBelowLimitOfQuantification(self): + """Returns whether the result is below the Limit of Quantification LOQ + """ + result = self.getResult() + if not api.is_floatable(result): + return False + + lloq = self.getLowerLimitOfQuantification() + return api.to_float(result) < api.to_float(lloq) + + @security.public + def isAboveLimitOfQuantification(self): + """Returns whether the result is above the Limit of Quantification LOQ + """ + result = self.getResult() + if not api.is_floatable(result): + return False + + uloq = self.getUpperLimitOfQuantification() + return api.to_float(result) > api.to_float(uloq) + + @security.public + def isOutsideTheQuantifiableRange(self): + """Returns whether the result falls outside the quantifiable range + specified by the Lower Limit of Quantification (LLOQ) and Upper Limit + of Quantification (ULOQ). + """ + if self.isBelowLimitOfQuantification(): + return True + if self.isAboveLimitOfQuantification(): + return True + return False + # TODO: REMOVE: nowhere used @deprecated("This Method will be removed in version 2.5") @security.public @@ -595,14 +649,24 @@ def calculateResult(self, override=False, cascade=False): key = dependency.getKeyword() ldl = dependency.getLowerDetectionLimit() udl = dependency.getUpperDetectionLimit() + lloq = dependency.getLowerLimitOfQuantification() + uloq = dependency.getUpperLimitOfQuantification() bdl = dependency.isBelowLowerDetectionLimit() adl = dependency.isAboveUpperDetectionLimit() + bloq = dependency.isBelowLimitOfQuantification() + aloq = dependency.isAboveLimitOfQuantification() mapping[key] = result mapping['%s.%s' % (key, 'RESULT')] = result mapping['%s.%s' % (key, 'LDL')] = api.to_float(ldl, 0.0) mapping['%s.%s' % (key, 'UDL')] = api.to_float(udl, 0.0) + mapping['%s.%s' % (key, 'LOQ')] = api.to_float(lloq, 0.0) + mapping['%s.%s' % (key, 'LLOQ')] = api.to_float(lloq, 0.0) + mapping['%s.%s' % (key, 'ULOQ')] = api.to_float(uloq, 0.0) mapping['%s.%s' % (key, 'BELOWLDL')] = int(bdl) mapping['%s.%s' % (key, 'ABOVEUDL')] = int(adl) + mapping['%s.%s' % (key, 'BELOWLOQ')] = int(bloq) + mapping['%s.%s' % (key, 'BELOWLLOQ')] = int(bloq) + mapping['%s.%s' % (key, 'ABOVEULOQ')] = int(aloq) except (TypeError, ValueError): return False @@ -787,66 +851,6 @@ def getRawAllowedInstruments(self): return [] return service.getRawInstruments() - @security.public - def getExponentialFormatPrecision(self, result=None): - """ Returns the precision for the Analysis Service and result - provided. Results with a precision value above this exponential - format precision should be formatted as scientific notation. - - If the Calculate Precision according to Uncertainty is not set, - the method will return the exponential precision value set in the - Schema. Otherwise, will calculate the precision value according to - the Uncertainty and the result. - - If Calculate Precision from the Uncertainty is set but no result - provided neither uncertainty values are set, returns the fixed - exponential precision. - - Will return positive values if the result is below 0 and will return - 0 or positive values if the result is above 0. - - Given an analysis service with fixed exponential format - precision of 4: - Result Uncertainty Returns - 5.234 0.22 0 - 13.5 1.34 1 - 0.0077 0.008 -3 - 32092 0.81 4 - 456021 423 5 - - For further details, visit https://jira.bikalabs.com/browse/LIMS-1334 - - :param result: if provided and "Calculate Precision according to the - Uncertainty" is set, the result will be used to retrieve the - uncertainty from which the precision must be calculated. Otherwise, - the fixed-precision will be used. - :returns: the precision - """ - if not result or self.getPrecisionFromUncertainty() is False: - return self._getExponentialFormatPrecision() - else: - uncertainty = self.getUncertainty(result) - if uncertainty is None: - return self._getExponentialFormatPrecision() - - try: - float(result) - except ValueError: - # if analysis result is not a number, then we assume in range - return self._getExponentialFormatPrecision() - - return get_significant_digits(uncertainty) - - def _getExponentialFormatPrecision(self): - field = self.getField('ExponentialFormatPrecision') - value = field.get(self) - if value is None: - # https://github.com/bikalims/bika.lims/issues/2004 - # We require the field, because None values make no sense at all. - value = self.Schema().getField( - 'ExponentialFormatPrecision').getDefault(self) - return value - @security.public def getFormattedResult(self, specs=None, decimalmark='.', sciformat=1, html=True): @@ -951,31 +955,40 @@ def getFormattedResult(self, specs=None, decimalmark='.', sciformat=1, fdm = formatDecimalMark('> %s' % hidemax, decimalmark) return fdm.replace('> ', '> ', 1) if html else fdm - # If below LDL, return '< LDL' - ldl = self.getLowerDetectionLimit() - ldl = api.to_float(ldl, 0.0) - if result < ldl: - # LDL must not be formatted according to precision, etc. - ldl = api.float_to_string(ldl) - fdm = formatDecimalMark('< %s' % ldl, decimalmark) - return fdm.replace('< ', '< ', 1) if html else fdm - - # If above UDL, return '< UDL' - udl = self.getUpperDetectionLimit() - udl = api.to_float(udl, 0.0) - if result > udl: - # UDL must not be formatted according to precision, etc. - udl = api.float_to_string(udl) - fdm = formatDecimalMark('> %s' % udl, decimalmark) - return fdm.replace('> ', '> ', 1) if html else fdm + # Lower Limits of Detection and Quantification (LLOD and LLOQ) + llod = api.to_float(self.getLowerDetectionLimit()) + lloq = api.to_float(self.getLowerLimitOfQuantification()) + if result < llod: + if llod != lloq: + # Display "Not detected" + result = t(_("result_below_llod", default="Not detected")) + return cgi.escape(result) if html else result + + # Display < LLOD + ldl = api.float_to_string(llod) + result = formatDecimalMark("< %s" % ldl, decimalmark) + return cgi.escape(result) if html else result + + if result < lloq: + lloq = api.float_to_string(lloq) + lloq = formatDecimalMark(lloq, decimalmark) + result = t(_("result_below_lloq", default="Detected but < ${LLOQ}", + mapping={"LLOQ": lloq})) + return cgi.escape(result) if html else result + + # Upper Limit of Quantification (ULOQ) + uloq = api.to_float(self.getUpperLimitOfQuantification()) + if result > uloq: + uloq = api.float_to_string(uloq) + result = formatDecimalMark('> %s' % uloq, decimalmark) + return cgi.escape(result) if html else result # Render numerical values - return format_numeric_result(self, self.getResult(), - decimalmark=decimalmark, + return format_numeric_result(self, decimalmark=decimalmark, sciformat=sciformat) @security.public - def getPrecision(self, result=None): + def getPrecision(self): """Returns the precision for the Analysis. - If ManualUncertainty is set, calculates the precision of the result @@ -997,20 +1010,27 @@ def getPrecision(self, result=None): Further information at AbstractBaseAnalysis.getPrecision() """ + precision = self.getField("Precision").get(self) allow_manual = self.getAllowManualUncertainty() precision_unc = self.getPrecisionFromUncertainty() if allow_manual or precision_unc: - uncertainty = self.getUncertainty(result) - if uncertainty is None: - return self.getField("Precision").get(self) - if api.to_float(uncertainty) == 0 and result is None: - return self.getField("Precision").get(self) - if api.to_float(uncertainty) == 0: - strres = str(result) - numdecimals = strres[::-1].find('.') - return numdecimals + + # if no uncertainty rely on the fixed precision + uncertainty = self.getUncertainty() + if not api.is_floatable(uncertainty): + return precision uncertainty = api.to_float(uncertainty) + if uncertainty == 0: + # calculate the precision from the result + try: + result = str(float(self.getResult())) + num_decimals = result[::-1].find('.') + return num_decimals + except ValueError: + # result not floatable, return the fixed precision + return precision + # Get the 'raw' significant digits from uncertainty sig_digits = get_significant_digits(uncertainty) # Round the uncertainty to its significant digit. @@ -1026,7 +1046,7 @@ def getPrecision(self, result=None): # Return the significant digit to apply return get_significant_digits(uncertainty) - return self.getField('Precision').get(self) + return precision @security.public def getAnalyst(self): diff --git a/src/bika/lims/content/abstractbaseanalysis.py b/src/bika/lims/content/abstractbaseanalysis.py index 7015316349..c5ed61c35a 100644 --- a/src/bika/lims/content/abstractbaseanalysis.py +++ b/src/bika/lims/content/abstractbaseanalysis.py @@ -175,37 +175,98 @@ ) ) -# If the value is below this limit, it means that the measurement lacks -# accuracy and this will be shown in manage_results and also on the final -# report. LowerDetectionLimit = StringField( "LowerDetectionLimit", - schemata="Analysis", + schemata="Limits", + default="0.0", + validators=("lower_limit_of_detection_validator",), + widget=DecimalWidget( + label=_( + u"label_analysis_lower_limit_of_detection_title", + default=u"Lower Limit of Detection (LLOD)" + ), + description=_( + u"label_analysis_lower_limit_of_detection_description", + default=u"The Lower Limit of Detection (LLOD) is the lowest " + u"concentration of a parameter that can be reliably " + u"detected by a specified testing methodology with a " + u"defined level of confidence. Results below this " + u"threshold are typically reported as '< LLOD' (or 'Not " + u"Detected'), indicating that the parameter's " + u"concentration, if present, is below the detection " + u"capability of the method at a reliable level." + ) + ) +) + +LowerLimitOfQuantification = StringField( + "LowerLimitOfQuantification", + schemata="Limits", default="0.0", + validators=("lower_limit_of_quantification_validator",), + widget=DecimalWidget( + label=_( + u"label_analysis_lower_limit_of_quantification_title", + default=u"Lower Limit Of Quantification (LLOQ)" + ), + description=_( + u"label_analysis_lower_limit_of_quantification_description", + default=u"The Lower Limit of Quantification (LLOQ) is the lowest " + u"concentration of a parameter that can be reliably and " + u"accurately measured using the specified testing " + u"methodology, with acceptable levels of precision and " + u"accuracy. Results below this value cannot be quantified " + u"with confidence and are typically reported as '< LOQ' " + u"(or 'Detected but < LOQ'), indicating that while the " + u"parameter may be present, its exact concentration " + u"cannot be determined reliably." + ) + ) +) + +UpperLimitOfQuantification = StringField( + "UpperLimitOfQuantification", + schemata="Limits", + default="1000000000.0", + validators=("upper_limit_of_quantification_validator",), widget=DecimalWidget( - label=_("Lower Detection Limit (LDL)"), + label=_( + u"label_analysis_upper_limit_of_quantification_title", + default=u"Upper Limit Of Quantification (ULOQ)"), description=_( - "The Lower Detection Limit is the lowest value to which the " - "measured parameter can be measured using the specified testing " - "methodology. Results entered which are less than this value will " - "be reported as < LDL") + u"label_analysis_upper_limit_of_quantification_description", + default=u"The Upper Limit of Quantification (ULOQ) is the highest " + u"concentration of a parameter that can be reliably and " + u"accurately measured using the specified testing " + u"methodology, with acceptable levels of precision and " + u"accuracy. Results above this value cannot be quantified " + u"with confidence and are typically reported as '> ULOQ', " + u"indicating that its exact concentration cannot be " + u"determined reliably." + ) ) ) -# If the value is above this limit, it means that the measurement lacks -# accuracy and this will be shown in manage_results and also on the final -# report. UpperDetectionLimit = StringField( "UpperDetectionLimit", - schemata="Analysis", + schemata="Limits", default="1000000000.0", widget=DecimalWidget( - label=_("Upper Detection Limit (UDL)"), + label=_( + u"label_analysis_upper_limit_of_detection_title", + default=u"Upper Limit of Detection (ULOD)"), description=_( - "The Upper Detection Limit is the highest value to which the " - "measured parameter can be measured using the specified testing " - "methodology. Results entered which are greater than this value " - "will be reported as > UDL") + u"label_analysis_upper_limit_of_detection_description", + default=u"The Upper Limit of Detection (ULOD) is the highest " + u"concentration of a parameter that can be reliably " + u"measured using a specified testing methodology. Beyond " + u"this limit, results may no longer be accurate or valid " + u"due to instrument saturation or methodological " + u"limitations. Results exceeding this threshold are " + u"typically reported as '> ULOD', indicating that the " + u"parameter's concentration is above the reliable " + u"detection range of the method." + ) ) ) @@ -230,7 +291,7 @@ # displayed in the results table. DetectionLimitSelector = BooleanField( 'DetectionLimitSelector', - schemata="Analysis", + schemata="Limits", default=False, widget=BooleanWidget( label=_("Display a Detection Limit selector"), @@ -249,7 +310,7 @@ # further information. AllowManualDetectionLimit = BooleanField( 'AllowManualDetectionLimit', - schemata="Analysis", + schemata="Limits", default=False, widget=BooleanWidget( label=_("Allow Manual Detection Limit input"), @@ -804,6 +865,8 @@ Precision, ExponentialFormatPrecision, LowerDetectionLimit, + LowerLimitOfQuantification, + UpperLimitOfQuantification, UpperDetectionLimit, DetectionLimitSelector, AllowManualDetectionLimit, @@ -962,6 +1025,24 @@ def getUpperDetectionLimit(self): value = value.rstrip("0").rstrip(".") return value + @security.public + def setLowerLimitOfQuantification(self, value): + """Sets the Lower Limit of Quantification (LLOQ) and ensures its value + is stored as a string without exponential notation and with whole + fraction preserved + """ + value = api.float_to_string(value) + self.getField("LowerLimitOfQuantification").set(self, value) + + @security.public + def setUpperLimitOfQuantification(self, value): + """Sets the Upper Limit of Quantification (ULOW) and ensures its value + is stored as a string without exponential notation and with whole + fraction preserved + """ + value = api.float_to_string(value) + self.getField("UpperLimitOfQuantification").set(self, value) + @security.public def isSelfVerificationEnabled(self): """Returns if the user that submitted a result for this analysis must diff --git a/src/bika/lims/utils/analysis.py b/src/bika/lims/utils/analysis.py index e81a19f525..02e494f9bf 100644 --- a/src/bika/lims/utils/analysis.py +++ b/src/bika/lims/utils/analysis.py @@ -237,10 +237,23 @@ def format_uncertainty(analysis, decimalmark=".", sciformat=1): By default 1 :returns: the formatted uncertainty """ - try: - result = float(analysis.getResult()) - except (ValueError, TypeError): - pass + if not api.is_floatable(analysis.getResult()): + # do not display uncertainty, result is not floatable + return "" + + if analysis.isOutsideTheQuantifiableRange(): + # Displaying uncertainty for results outside the quantifiable range is + # not meaningful because the Lower Limit of Quantification (LLOQ) and + # Upper Limit of Quantification (ULOQ) define the range within which + # a parameter can be reliably and accurately measured. Results outside + # this range are prone to significant variability and may be + # indistinguishable from background noise or method imprecision. + # As such, any numeric value reported outside the quantifiable range + # lacks the reliability required for meaningful interpretation. + # It is important to note that the quantifiable range is always nested + # within the detection range, which is defined by the Lower Limit of + # Detection (LLOD) and Upper Limit of Detection (ULOD). + return "" uncertainty = analysis.getUncertainty() if api.to_float(uncertainty, default=-1) < 0: @@ -260,7 +273,7 @@ def format_uncertainty(analysis, decimalmark=".", sciformat=1): precision = uncertainty[::-1].find(".") if precision == -1: - precision = analysis.getPrecision(result) + precision = analysis.getPrecision() # Scientific notation? # Get the default precision for scientific notation @@ -276,7 +289,7 @@ def format_uncertainty(analysis, decimalmark=".", sciformat=1): return formatDecimalMark(formatted, decimalmark) -def format_numeric_result(analysis, result, decimalmark='.', sciformat=1): +def format_numeric_result(analysis, decimalmark='.', sciformat=1): """ Returns the formatted number part of a results value. This is responsible for deciding the precision, and notation of numeric @@ -332,6 +345,7 @@ def format_numeric_result(analysis, result, decimalmark='.', sciformat=1): :result: should be a string to preserve the decimal precision. :returns: the formatted result as string """ + result = analysis.getResult() try: result = float(result) except ValueError: @@ -344,7 +358,7 @@ def format_numeric_result(analysis, result, decimalmark='.', sciformat=1): # Scientific notation? # Get the default precision for scientific notation threshold = analysis.getExponentialFormatPrecision() - precision = analysis.getPrecision(result) + precision = analysis.getPrecision() formatted = _format_decimal_or_sci(result, precision, threshold, sciformat) return formatDecimalMark(formatted, decimalmark) diff --git a/src/bika/lims/validators.py b/src/bika/lims/validators.py index d696491488..d1dcfeb042 100644 --- a/src/bika/lims/validators.py +++ b/src/bika/lims/validators.py @@ -28,13 +28,13 @@ from bika.lims import logger from bika.lims.api import APIError from bika.lims.catalog import SETUP_CATALOG -from senaite.core.i18n import translate as _t from bika.lims.utils import to_utf8 from Products.CMFCore.utils import getToolByName from Products.CMFPlone.utils import safe_unicode from Products.validation import validation from Products.validation.interfaces.IValidator import IValidator from Products.ZCTextIndex.ParseTree import ParseError +from senaite.core.i18n import translate as _t from zope.interface import implements @@ -501,8 +501,13 @@ def __call__(self, value, *args, **kwargs): }) return to_utf8(translate(msg)) - # Allow to use Wildcards, LDL and UDL values in calculations - allowedwds = ["LDL", "UDL", "BELOWLDL", "ABOVEUDL"] + # Allow to use Wildcards, LDL, UDL and LLOQ values in calculations + allowedwds = [ + "LDL", "BELOWLDL", + "UDL", "ABOVEUDL", + "LOQ", "LLOQ", "BELOWLOQ", "BELOWLLOQ", + "ULOQ", "ABOVEULOQ", + ] keysandwildcards = re.compile(r"\[([^\]]+)\]").findall(value) keysandwildcards = [k for k in keysandwildcards if "." in k] keysandwildcards = [k.split(".", 1) for k in keysandwildcards] @@ -1456,3 +1461,90 @@ def validate_record(self, record): validation.register(ServiceConditionsValidator()) + + +class LowerLimitOfDetectionValidator(object): + """Validates that the Lower Limit of Detection (LLOD) is lower than or + equal to the Lower Limit of Quantification (LLOQ) + """ + implements(IValidator) + name = "lower_limit_of_detection_validator" + + def __call__(self, value, **kwargs): + instance = kwargs["instance"] + field_name = kwargs["field"].getName() + + # get the value (or fallback to field's default) + default = instance.getField(field_name).getDefault(instance) + llod = api.to_float(value, default) + + form = kwargs["REQUEST"].form + lloq = form.get("LowerLimitOfQuantification", None) + lloq = api.to_float(lloq, llod) + if llod > lloq: + return _t(_( + u"validator_llod_above_lloq", + default=u"The Lower Limit of Detection (LLOD) cannot be " + u"greater than the Lower Limit of Quantification " + u"(LLOQ)." + )) + + +class LowerLimitOfQuantificationValidator(object): + """Validates that the Lower Limit of Quantification (LLOQ) is lower than + the Upper Limit of Quantification (ULOQ) + """ + implements(IValidator) + name = "lower_limit_of_quantification_validator" + + def __call__(self, value, **kwargs): + instance = kwargs["instance"] + field_name = kwargs["field"].getName() + + # get the value (or fallback to field's default) + default = instance.getField(field_name).getDefault(instance) + lloq = api.to_float(value, default) + + # compare with the lower limit of detection + form = kwargs["REQUEST"].form + uloq = form.get("UpperLimitOfQuantification", None) + uloq = api.to_float(uloq, lloq) + if lloq >= uloq: + return _t(_( + u"validator_lloq_above_uloq", + default=u"The Lower Limit of Quantification (LLOQ) cannot be " + u"greater than or equal to the Upper Limit of " + u"Quantification (ULOQ)." + )) + + +class UpperLimitOfQuantificationValidator(object): + """Validates that the Upper Limit of Quantification (ULOD) is lower than + or equal to the Upper Limit of Detection (ULOD) + """ + implements(IValidator) + name = "upper_limit_of_quantification_validator" + + def __call__(self, value, **kwargs): + instance = kwargs["instance"] + field_name = kwargs["field"].getName() + + # get the value (or fallback to field's default) + default = instance.getField(field_name).getDefault(instance) + uloq = api.to_float(value, default) + + # compare with the lower limit of detection + form = kwargs["REQUEST"].form + ulod = form.get("UpperDetectionLimit", None) + ulod = api.to_float(ulod, uloq) + if uloq > ulod: + return _t(_( + u"validator_uloq_above_ulod", + default=u"The Upper Limit of Quantification (LLOQ) cannot be " + u"greater than the Upper Limit of Detection (ULOD)." + )) + + +validation.register(LowerLimitOfDetectionValidator()) +validation.register(LowerLimitOfQuantificationValidator()) +validation.register(UpperLimitOfQuantificationValidator()) diff --git a/src/senaite/core/profiles/default/metadata.xml b/src/senaite/core/profiles/default/metadata.xml index a5a47b80b7..1ba71f2343 100644 --- a/src/senaite/core/profiles/default/metadata.xml +++ b/src/senaite/core/profiles/default/metadata.xml @@ -1,6 +1,6 @@ - 2658 + 2659 profile-Products.ATContentTypes:base profile-Products.CMFEditions:CMFEditions diff --git a/src/senaite/core/tests/doctests/Limits.rst b/src/senaite/core/tests/doctests/Limits.rst new file mode 100644 index 0000000000..99457eb08f --- /dev/null +++ b/src/senaite/core/tests/doctests/Limits.rst @@ -0,0 +1,334 @@ +Limits +------ + +It is possible to assign Limits of Detection and Quantification to services and +analyses. The way results are formatted and displayed for visualization and in +reports depend on these limits. + +Running this test from the buildout directory:: + + bin/test test_textual_doctests -t Limits + + +Test Setup +.......... + +Needed Imports: + + >>> from bika.lims import api + >>> from bika.lims.utils.analysisrequest import create_analysisrequest + >>> from bika.lims.workflow import doActionFor as do_action_for + >>> from DateTime import DateTime + >>> from plone.app.testing import setRoles + >>> from plone.app.testing import TEST_USER_ID + >>> from plone.app.testing import TEST_USER_PASSWORD + +Functional Helpers: + + >>> def new_sample(services): + ... values = { + ... 'Client': client, + ... 'Contact': contact, + ... 'DateSampled': date_now, + ... 'SampleType': sampletype + ... } + ... sample = create_analysisrequest(client, request, values, services) + ... transitioned = do_action_for(sample, "receive") + ... return sample + + >>> def get_analysis(sample, service): + ... service_uid = api.get_uid(service) + ... for analysis in sample.getAnalyses(full_objects=True): + ... if analysis.getServiceUID() == service_uid: + ... return analysis + ... return None + +Variables: + + >>> portal = self.portal + >>> request = self.request + >>> setup = portal.setup + >>> bika_setup = portal.bika_setup + >>> date_now = DateTime().strftime("%Y-%m-%d") + +Create some basic objects for the test: + + >>> setRoles(portal, TEST_USER_ID, ['LabManager',]) + >>> client = api.create(portal.clients, "Client", Name="Happy Hills", ClientID="HH") + >>> contact = api.create(client, "Contact", Firstname="Rita", Lastname="Mohale") + >>> sampletype = api.create(setup.sampletypes, "SampleType", title="Water", Prefix="W") + >>> labcontact = api.create(bika_setup.bika_labcontacts, "LabContact", Firstname="Lab", Lastname="Manager") + >>> department = api.create(setup.departments, "Department", title="Chemistry", Manager=labcontact) + >>> category = api.create(setup.analysiscategories, "AnalysisCategory", title="Metals", Department=department) + >>> Cu = api.create(bika_setup.bika_analysisservices, "AnalysisService", title="Copper", Keyword="Cu", Category=category) + + +Test detection range +.................... + +We allow users to input the Lower Limit of Detection (LLOD) and the Upper +Limit of Detection (ULOD), which together define the range within which the +concentration of the analyte can be reliably **detected**: + + >>> Cu.setLowerDetectionLimit("10") + >>> Cu.setUpperDetectionLimit("20") + +Create a sample: + + >>> sample = new_sample([Cu]) + >>> cu = get_analysis(sample, Cu) + +The result is formatted as `< LLOD` when below the Lower Limit of Detection +(LLOD): + + >>> cu.setResult(5) + >>> cu.isBelowLowerDetectionLimit() + True + >>> cu.isAboveUpperDetectionLimit() + False + >>> cu.getFormattedResult() + '< 10' + >>> cu.getFormattedResult(html=False) + '< 10' + +The result is not formatted when equals to LLOD: + + >>> cu.setResult(10) + >>> cu.isBelowLowerDetectionLimit() + False + >>> cu.isAboveUpperDetectionLimit() + False + >>> cu.getFormattedResult() + '10' + +The result is not formatted when equals to ULOD: + + >>> cu.setResult(20) + >>> cu.isBelowLowerDetectionLimit() + False + >>> cu.isAboveUpperDetectionLimit() + False + >>> cu.getFormattedResult() + '20' + +The result is formatted as `> ULOD` when above the Upper Limit of Detection +(ULOD): + + >>> cu.setResult(25) + >>> cu.isBelowLowerDetectionLimit() + False + >>> cu.isAboveUpperDetectionLimit() + True + >>> cu.getFormattedResult() + '> 20' + >>> cu.getFormattedResult(html=False) + '> 20' + + +Test quantifiable range +....................... + +We allow users to input the Lower Limit of Quantification (LLOQ) and the Upper +Limit of Quantification (ULOQ), which together define the range within which +the concentration of the analyte can be reliably **quantified**. + +We can set the same range for quantifiable as for detection: + + >>> Cu.setLowerDetectionLimit("10") + >>> Cu.setUpperDetectionLimit("20") + >>> Cu.setLowerLimitOfQuantification("10") + >>> Cu.setUpperLimitOfQuantification("20") + +Create a sample: + + >>> sample = new_sample([Cu]) + >>> cu = get_analysis(sample, Cu) + +The result is formatted as `< LLOQ` when below the Lower Limit of +Quantification (LLOQ): + + >>> cu.setResult(5) + >>> cu.isBelowLowerDetectionLimit() + True + >>> cu.isAboveUpperDetectionLimit() + False + >>> cu.isOutsideTheQuantifiableRange() + True + >>> cu.getFormattedResult() + '< 10' + >>> cu.getFormattedResult(html=False) + '< 10' + +The result is not formatted when equals to LLOQ: + + >>> cu.setResult(10) + >>> cu.isBelowLowerDetectionLimit() + False + >>> cu.isAboveUpperDetectionLimit() + False + >>> cu.isOutsideTheQuantifiableRange() + False + >>> cu.getFormattedResult() + '10' + +The result is not formatted when equals to ULOQ: + + >>> cu.setResult(20) + >>> cu.isBelowLowerDetectionLimit() + False + >>> cu.isAboveUpperDetectionLimit() + False + >>> cu.isOutsideTheQuantifiableRange() + False + >>> cu.getFormattedResult() + '20' + +The result is formatted as `> ULOQ` when above the Upper Limit of +Quantification (ULOQ): + + >>> cu.setResult(25) + >>> cu.isBelowLowerDetectionLimit() + False + >>> cu.isAboveUpperDetectionLimit() + True + >>> cu.isOutsideTheQuantifiableRange() + True + >>> cu.getFormattedResult() + '> 20' + >>> cu.getFormattedResult(html=False) + '> 20' + + +Test quantifiable and detection range altogether +................................................ + +Set different ranges for quantifiable and detection. Note that the +quantifiable range is always nested within the detection range:. + + >>> Cu.setLowerDetectionLimit("5") + >>> Cu.setUpperDetectionLimit("25") + >>> Cu.setLowerLimitOfQuantification("10") + >>> Cu.setUpperLimitOfQuantification("20") + +Create a sample: + + >>> sample = new_sample([Cu]) + >>> cu = get_analysis(sample, Cu) + +The result is formatted as `Not detected` when below the LLOD: + + >>> cu.setResult(2) + >>> cu.isBelowLowerDetectionLimit() + True + >>> cu.isAboveUpperDetectionLimit() + False + >>> cu.isOutsideTheQuantifiableRange() + True + >>> cu.getFormattedResult() + 'Not detected' + +The result is formatted as `Detected but < LLOQ` when equal to LLOD: + + >>> cu.setResult(5) + >>> cu.isBelowLowerDetectionLimit() + False + >>> cu.isAboveUpperDetectionLimit() + False + >>> cu.isOutsideTheQuantifiableRange() + True + >>> cu.getFormattedResult() + 'Detected but < 10' + >>> cu.getFormattedResult(html=False) + 'Detected but < 10' + +Result is formatted as `Detected but < LLOQ` when above LLOD but below LLOQ: + + >>> cu.setResult(5) + >>> cu.isBelowLowerDetectionLimit() + False + >>> cu.isAboveUpperDetectionLimit() + False + >>> cu.isOutsideTheQuantifiableRange() + True + >>> cu.getFormattedResult() + 'Detected but < 10' + >>> cu.getFormattedResult(html=False) + 'Detected but < 10' + +The result is not formatted when equal to LLOQ: + + >>> cu.setResult(10) + >>> cu.isBelowLowerDetectionLimit() + False + >>> cu.isAboveUpperDetectionLimit() + False + >>> cu.isOutsideTheQuantifiableRange() + False + >>> cu.getFormattedResult() + '10' + +The result is not formatted when within quantifiable range: + + >>> cu.setResult(15) + >>> cu.isBelowLowerDetectionLimit() + False + >>> cu.isAboveUpperDetectionLimit() + False + >>> cu.isOutsideTheQuantifiableRange() + False + >>> cu.getFormattedResult() + '15' + +The result is not formatted when equal to ULOQ: + + >>> cu.setResult(20) + >>> cu.isBelowLowerDetectionLimit() + False + >>> cu.isAboveUpperDetectionLimit() + False + >>> cu.isOutsideTheQuantifiableRange() + False + >>> cu.getFormattedResult() + '20' + +The result is formatted as `> ULOQ` when above ULOQ but below ULOD: + + >>> cu.setResult(22) + >>> cu.isBelowLowerDetectionLimit() + False + >>> cu.isAboveUpperDetectionLimit() + False + >>> cu.isOutsideTheQuantifiableRange() + True + >>> cu.getFormattedResult() + '> 20' + >>> cu.getFormattedResult(html=False) + '> 20' + +The result is formatted as `> ULOQ` when equals ULOD: + + >>> cu.setResult(25) + >>> cu.isBelowLowerDetectionLimit() + False + >>> cu.isAboveUpperDetectionLimit() + False + >>> cu.isOutsideTheQuantifiableRange() + True + >>> cu.getFormattedResult() + '> 20' + >>> cu.getFormattedResult(html=False) + '> 20' + +The result is formatted as `> ULOQ` when above ULOD: + + >>> cu.setResult(30) + >>> cu.isBelowLowerDetectionLimit() + False + >>> cu.isAboveUpperDetectionLimit() + True + >>> cu.isOutsideTheQuantifiableRange() + True + >>> cu.getFormattedResult() + '> 20' + >>> cu.getFormattedResult(html=False) + '> 20' diff --git a/src/senaite/core/tests/doctests/Uncertainties.rst b/src/senaite/core/tests/doctests/Uncertainties.rst index 041cbae0a4..490df0a14b 100644 --- a/src/senaite/core/tests/doctests/Uncertainties.rst +++ b/src/senaite/core/tests/doctests/Uncertainties.rst @@ -329,6 +329,161 @@ Check uncertainty when the result is exactly on a detection limit: '0.001' +Test uncertainty for results above/below quantifiable range +........................................................... + +Setup uncertainty settings in the service: + + >>> Cu.setAllowManualUncertainty(True) + >>> Cu.setUncertainties(uncertainties) + >>> Cu.setPrecisionFromUncertainty(True) + >>> Cu.setLowerDetectionLimit("10") + >>> Cu.setLowerLimitOfQuantification("15") + >>> Cu.setUpperLimitOfQuantification("25") + >>> Cu.setUpperDetectionLimit("30") + +Create the sample with the analysis: + + >>> sample = new_sample([Cu]) + >>> cu = get_analysis(sample, Cu) + +Test with a result below both LLOQ and LLOD: + + >>> cu.setResult("9") + >>> cu.isOutsideTheQuantifiableRange() + True + >>> cu.isBelowLowerDetectionLimit() + True + >>> cu.isBelowLimitOfQuantification() + True + >>> cu.isAboveLimitOfQuantification() + False + >>> cu.isAboveUpperDetectionLimit() + False + >>> cu.getUncertainty() is None + True + +Test with a result that equals LLOD: + + >>> cu.setResult("10") + >>> cu.isOutsideTheQuantifiableRange() + True + >>> cu.isBelowLowerDetectionLimit() + False + >>> cu.isBelowLimitOfQuantification() + True + >>> cu.isAboveLimitOfQuantification() + False + >>> cu.isAboveUpperDetectionLimit() + False + >>> cu.getUncertainty() is None + True + +Test with a result above LLOD, but below LLOQ: + + >>> cu.setResult("12.5") + >>> cu.isOutsideTheQuantifiableRange() + True + >>> cu.getUncertainty() is None + True + +Test with a result that equals to LLOQ: + + >>> cu.setResult("15") + >>> cu.isOutsideTheQuantifiableRange() + False + >>> cu.isBelowLowerDetectionLimit() + False + >>> cu.isBelowLimitOfQuantification() + False + >>> cu.isAboveLimitOfQuantification() + False + >>> cu.isAboveUpperDetectionLimit() + False + >>> cu.getUncertainty() + '0.4' + +Test with a result above LLOQ, but below ULOQ: + + >>> cu.setResult("20") + >>> cu.isOutsideTheQuantifiableRange() + False + >>> cu.isBelowLowerDetectionLimit() + False + >>> cu.isBelowLimitOfQuantification() + False + >>> cu.isAboveLimitOfQuantification() + False + >>> cu.isAboveUpperDetectionLimit() + False + >>> cu.getUncertainty() + '0.4' + +Test with a result that equals ULOQ: + + >>> cu.setResult("25") + >>> cu.isOutsideTheQuantifiableRange() + False + >>> cu.isBelowLowerDetectionLimit() + False + >>> cu.isBelowLimitOfQuantification() + False + >>> cu.isAboveLimitOfQuantification() + False + >>> cu.isAboveUpperDetectionLimit() + False + >>> cu.getUncertainty() is None + True + +Test with a result above ULOQ, but below ULOD: + + >>> cu.setResult("27.5") + >>> cu.isOutsideTheQuantifiableRange() + True + >>> cu.isBelowLowerDetectionLimit() + False + >>> cu.isBelowLimitOfQuantification() + False + >>> cu.isAboveLimitOfQuantification() + True + >>> cu.isAboveUpperDetectionLimit() + False + >>> cu.getUncertainty() is None + True + +Test with a result that equals ULOD: + + >>> cu.setResult("30") + >>> cu.isOutsideTheQuantifiableRange() + True + >>> cu.isBelowLowerDetectionLimit() + False + >>> cu.isBelowLimitOfQuantification() + False + >>> cu.isAboveLimitOfQuantification() + True + >>> cu.isAboveUpperDetectionLimit() + False + >>> cu.getUncertainty() is None + True + +Test with a result above both ULLOQ and ULOD: + + >>> cu.setResult("35") + >>> cu.isOutsideTheQuantifiableRange() + True + >>> cu.isBelowLowerDetectionLimit() + False + >>> cu.isBelowLimitOfQuantification() + False + >>> cu.isAboveLimitOfQuantification() + True + >>> cu.isAboveUpperDetectionLimit() + True + >>> cu.getUncertainty() is None + True + + Test uncertainty formatting ........................... @@ -537,3 +692,88 @@ Change to a higher precision threshold: >>> au.setExponentialFormatPrecision(30) >>> format_uncertainty(au) '0.000000000000000000001' + + +Test formatting with detection and quantification limits +........................................................ + +Displaying uncertainty for results outside the quantifiable range is not +meaningful because the Lower Limit of Quantification (LLOQ) and Upper Limit of +Quantification (ULOQ) define the range within which a parameter can be reliably +and accurately measured. Results outside this range are prone to significant +variability and may be indistinguishable from background noise or method +imprecision. As such, any numeric value reported outside the quantifiable rang +lacks the reliability required for meaningful interpretation. + +It is important to note that the quantifiable range is always nested within the +detection range, which is defined by the Lower Limit of Detection (LLOD) and +Upper Limit of Detection (ULOD). + +Create a new sample: + + >>> sample = new_sample([Cu]) + >>> cu = get_analysis(sample, Cu) + >>> cu.setAllowManualUncertainty(True) + >>> cu.setPrecisionFromUncertainty(False) + +Manually set the limit of quantification, limit of detection, and uncertainty: + + >>> cu.setUncertainty("0.00000123") + >>> cu.setLowerDetectionLimit("10") + >>> cu.setLowerLimitOfQuantification("15") + >>> cu.setUpperLimitOfQuantification("25") + >>> cu.setUpperDetectionLimit("30") + +Set a result below the Lower Limit of Detection (LLOD): + + >>> cu.setResult(5) + >>> format_uncertainty(cu) + '' + +Set a result that equals to the LLOD: + + >>> cu.setResult(10) + >>> format_uncertainty(cu) + '' + +Set a result below the Lower Limit of Quantification (LLOQ) but above LLOD: + + >>> cu.setResult(12) + >>> format_uncertainty(cu) + '' + +Set a result that equals to the LLOQ: + + >>> cu.setResult(15) + >>> format_uncertainty(cu) + '0.00000123' + +Set a result within the quantifiable range: + + >>> cu.setResult(20) + >>> format_uncertainty(cu) + '0.00000123' + +Set a result that equals to the ULOQ: + + >>> cu.setResult(25) + >>> format_uncertainty(cu) + '0.00000123' + +Set a result above ULOQ and below the Upper Limit of Detection (ULOD): + + >>> cu.setResult(28) + >>> format_uncertainty(cu) + '' + +Set a result that equals to ULOD: + + >>> cu.setResult(30) + >>> format_uncertainty(cu) + '' + +Set a result above ULOD: + + >>> cu.setResult(35) + >>> format_uncertainty(cu) + '' diff --git a/src/senaite/core/tests/test_decimal-sci-notation.py b/src/senaite/core/tests/test_decimal-sci-notation.py index a4a7335531..228a0e7cd3 100644 --- a/src/senaite/core/tests/test_decimal-sci-notation.py +++ b/src/senaite/core/tests/test_decimal-sci-notation.py @@ -318,7 +318,11 @@ def test_DecimalSciNotation(self): [2, 4, 5, '-12340.0123', '-1.234001·104'], ] s = self.service - s.setLowerDetectionLimit('-99999') # We want to test results below 0 too + + # Set negative values for LLOD and LOQ to test results below 0 + s.setLowerDetectionLimit('-99999') + s.setLowerLimitOfQuantification('-99999') + prevm = [] an = None bs = self.portal.setup diff --git a/src/senaite/core/tests/test_decimalmark-sci-notation.py b/src/senaite/core/tests/test_decimalmark-sci-notation.py index c73c5e0e3c..45e0c5704c 100644 --- a/src/senaite/core/tests/test_decimalmark-sci-notation.py +++ b/src/senaite/core/tests/test_decimalmark-sci-notation.py @@ -103,7 +103,11 @@ def test_DecimalMarkWithSciNotation(self): [4, 3, 5, '-1234.5678', '-1,2345678·103'], ] s = self.service - s.setLowerDetectionLimit('-99999') # We want to test results below 0 too + + # Set negative values for LLOD and LOQ to test results below 0 + s.setLowerDetectionLimit('-99999') + s.setLowerLimitOfQuantification('-99999') + prevm = [] an = None bs = self.portal.bika_setup diff --git a/src/senaite/core/tests/test_limitdetections.py b/src/senaite/core/tests/test_limitdetections.py index 84b76bdba9..500e06377c 100644 --- a/src/senaite/core/tests/test_limitdetections.py +++ b/src/senaite/core/tests/test_limitdetections.py @@ -326,6 +326,11 @@ def test_ar_manage_results_detectionlimit_selector_manual(self): s.setLowerDetectionLimit(case['min']) s.setUpperDetectionLimit(case['max']) + # Set same Limit of Quantification (LOQ) as the Lower Limit of + # Detection (LLOD/LDL). Otherwise, negative results will be + # formatted as `> LOQ` instead. + s.setLowerLimitOfQuantification(case['min']) + # Input results # Client: Happy Hills # SampleType: Apple Pulp diff --git a/src/senaite/core/upgrade/v02_06_000.py b/src/senaite/core/upgrade/v02_06_000.py index 58eef017a1..d6b09fe574 100644 --- a/src/senaite/core/upgrade/v02_06_000.py +++ b/src/senaite/core/upgrade/v02_06_000.py @@ -2747,3 +2747,45 @@ def reindex_sub_groups(tool): obj.sort_key = api.to_float(obj.sort_key, 0.0) obj.reindexObject(idxs=["sortable_title"], update_metadata=False) logger.info("Reindexing sub groups [DONE]") + + +def init_loq(tool): + """Initializes the value of the field LowerLimitOfQuantification with the + value of the Lower Limit of Detection (LLOD or LDL) to ensure LLOQ is + greater than or equal to LLOD + """ + logger.info("Initializing the Limit of Quantification (LOQ) ...") + + # Note there is no need to update analyses, cause on them, the function + # `abstractanalysis.getLowerLimitOfQuantification` returns the LLOD if the + # value set for LLOQ is not set or lower than LLOD. + + cat = api.get_tool(SETUP_CATALOG) + for brain in cat(portal_type="AnalysisService"): + obj = brain.getObject() + + # get the raw values + llod = obj.getField("LowerDetectionLimit").getRaw(obj) + lloq = obj.getField("LowerLimitOfQuantification").getRaw(obj) + uloq = obj.getField("UpperLimitOfQuantification").getRaw(obj) + ulod = obj.getField("UpperDetectionLimit").getRaw(obj) + + # convert values to float + fllod = api.to_float(llod) + flloq = api.to_float(lloq) + fuloq = api.to_float(uloq) + fulod = api.to_float(ulod) + + # ·······|-------|=======|-------|······· + # LLOD LLOQ ULOQ ULOD + if fllod <= flloq < fuloq <= fulod: + obj._p_deactivate() + continue + + # assign defaults + obj.setLowerLimitOfQuantification(llod) + obj.setUpperLimitOfQuantification(ulod) + obj.reindexObject() + obj._p_deactivate() + + logger.info("Initializing the Limit of Quantification (LOQ) [DONE]") diff --git a/src/senaite/core/upgrade/v02_06_000.zcml b/src/senaite/core/upgrade/v02_06_000.zcml index a5c0e56c3b..bb49d8fff6 100644 --- a/src/senaite/core/upgrade/v02_06_000.zcml +++ b/src/senaite/core/upgrade/v02_06_000.zcml @@ -3,6 +3,14 @@ xmlns:genericsetup="http://namespaces.zope.org/genericsetup" i18n_domain="senaite.core"> + +