add no skill to P/R curve graph artifact

Author: danabens

src/smexperiments/tracker.py

@@ -467,6 +467,7 @@ def log_precision_recall(
         positive_label=None,
         title=None,
         output_artifact=True,
+        no_skill=None,
     ):
         """Log a precision recall graph artifact which will be displayed in studio.
             Requires sklearn.  Not yet supported by studio.
@@ -479,8 +480,9 @@ def log_precision_recall(
 
                 y_true = [0, 0, 1, 1]
                 y_scores = [0.1, 0.4, 0.35, 0.8]
+                no_skill = len(y_true[y_true==1]) / len(y_true)
 
-                my_tracker._log_precision_recall(y_true, y_scores)
+                my_tracker._log_precision_recall(y_true, y_scores, no_skill=no_skill)
 
         Args:
             y_true (array): True labels. If labels are not binary then positive_label should be given.
@@ -489,6 +491,9 @@ def log_precision_recall(
             title (str, optional): Title of the graph, Defaults to none.
             output_artifact (boolean, optional): Determines if the artifact is associated with the
                 Trial Component as an output artifact. If False will be an input artifact.
+            no_skill (int): The precision threshold under which the classifier cannot discriminate
+                between the classes and would predict a random class or a constant class in
+                all cases.
 
         Raises:
             ValueError: If length mismatch between y_true and predicted_probabilities.
@@ -516,6 +521,7 @@ def log_precision_recall(
             "precision": precision.tolist(),
             "recall": recall.tolist(),
             "averagePrecisionScore": ap,
+            "noSkill": no_skill,
         }
         self._log_graph_artifact(title, data, "PrecisionRecallCurve", output_artifact)
 

tests/unit/artifact_schemas/precision_recall_curve_v0.json

@@ -28,6 +28,10 @@
     "averagePrecisionScore": {
       "description": "AP summarizes a precision-recall curve as the weighted mean of precisions achieved at each threshold, with the increase in recall from the previous threshold used as the weight.",
       "type": "number"
+    },
+    "noSkill": {
+      "description": "The precision threshold under which the classifier cannot discriminate between the classes and would predict a random class or a constant class in all cases.",
+      "type": "number"
     }
   },
   "additionalProperties": false

tests/unit/test_tracker.py

@@ -341,10 +341,11 @@ def test_log_pr_curve(under_test):
 
     y_true = [0, 0, 1, 1]
     y_scores = [0.1, 0.4, 0.35, 0.8]
+    no_skill = 0.1
 
     under_test._artifact_uploader.upload_object_artifact.return_value = ("s3uri_value", "etag_value")
 
-    under_test.log_precision_recall(y_true, y_scores, title="TestPRCurve")
+    under_test.log_precision_recall(y_true, y_scores, title="TestPRCurve", no_skill=no_skill)
 
     expected_data = {
         "type": "PrecisionRecallCurve",
@@ -353,6 +354,7 @@ def test_log_pr_curve(under_test):
         "precision": [0.6666666666666666, 0.5, 1.0, 1.0],
         "recall": [1.0, 0.5, 0.5, 0.0],
         "averagePrecisionScore": 0.8333333333333333,
+        "noSkill": 0.1,
     }
     under_test._artifact_uploader.upload_object_artifact.assert_called_with(
         "TestPRCurve", expected_data, file_extension="json"