[skipci] [Doctest] added contrib metrics AveragePrecision, PrecisionRecallCurve, ROC_AUC and RocCurve

docs/source/conf.py

@@ -374,6 +374,11 @@ def train_step(engine, batch):
     ('fc', nn.Linear(2, 1))
 ]))
 
+def sigmoid_output_transform(output):
+    y_pred, y = output
+    y_pred = torch.sigmoid(y_pred)
+    return y_pred, y
+
 manual_seed(666)
 """
 

ignite/contrib/metrics/average_precision.py

@@ -33,14 +33,23 @@ class AveragePrecision(EpochMetric):
         AveragePrecision expects y to be comprised of 0's and 1's. y_pred must either be probability estimates or
         confidence values. To apply an activation to y_pred, use output_transform as shown below:
 
-        .. code-block:: python
+        .. testcode::
 
             def activated_output_transform(output):
                 y_pred, y = output
                 y_pred = torch.softmax(y_pred, dim=1)
                 return y_pred, y
+            y_pred = torch.Tensor([[3.89, 0.21], [0.30, 0.9], [0.46, 0.14], [0.16, 0.2]])
+            y_true = torch.tensor([[1, 1], [1, 1], [1, 0], [0, 1]])
 
             avg_precision = AveragePrecision(activated_output_transform)
+            avg_precision.attach(default_evaluator, 'average_precision')
+            state = default_evaluator.run([[y_pred, y_true]])
+            print(state.metrics['average_precision'])
+
+        .. testoutput::
+
+            0.9166...
 
     """
 

ignite/contrib/metrics/precision_recall_curve.py

@@ -35,14 +35,27 @@ class PrecisionRecallCurve(EpochMetric):
     PrecisionRecallCurve expects y to be comprised of 0's and 1's. y_pred must either be probability estimates
     or confidence values. To apply an activation to y_pred, use output_transform as shown below:
 
-    .. code-block:: python
+    .. testcode::
 
         def activated_output_transform(output):
             y_pred, y = output
             y_pred = torch.sigmoid(y_pred)
             return y_pred, y
+        y_pred = torch.tensor([-3, 0.4, 0.9, 8])
+        y_true = torch.tensor([0, 0, 1, 1])
+        prec_recall_curve = PrecisionRecallCurve(activated_output_transform)
+        prec_recall_curve.attach(default_evaluator, 'prec_recall_curve')
+        state = default_evaluator.run([[y_pred, y_true]])
 
-        roc_auc = PrecisionRecallCurve(activated_output_transform)
+        print("Precision", [round(i, 4) for i in state.metrics['prec_recall_curve'][0].tolist()])
+        print("Recall", [round(i, 4) for i in state.metrics['prec_recall_curve'][1].tolist()])
+        print("Thresholds", [round(i, 4) for i in state.metrics['prec_recall_curve'][2].tolist()])
+
+    .. testoutput::
+
+        Precision [1.0, 1.0, 1.0]
+        Recall [1.0, 0.5, 0.0]
+        Thresholds [0.7109, 0.9997]
 
     """
 

ignite/contrib/metrics/roc_auc.py

@@ -41,15 +41,18 @@ class ROC_AUC(EpochMetric):
     ROC_AUC expects y to be comprised of 0's and 1's. y_pred must either be probability estimates or confidence
     values. To apply an activation to y_pred, use output_transform as shown below:
 
-    .. code-block:: python
+    .. testcode::
 
-        def activated_output_transform(output):
-            y_pred, y = output
-            y_pred = torch.sigmoid(y_pred)
-            return y_pred, y
+        roc_auc = ROC_AUC(sigmoid_output_transform)
+        roc_auc.attach(default_evaluator, 'roc_auc')
+        y_pred = torch.tensor([[-3], [0.4], [0.9], [8]])
+        y_true = torch.tensor([[0], [0], [1], [0]])
+        state = default_evaluator.run([[y_pred, y_true]])
+        print(state.metrics['roc_auc'])
 
-        roc_auc = ROC_AUC(activated_output_transform)
+    .. testoutput::
 
+        0.6666...
     """
 
     def __init__(
@@ -88,15 +91,22 @@ class RocCurve(EpochMetric):
     RocCurve expects y to be comprised of 0's and 1's. y_pred must either be probability estimates or confidence
     values. To apply an activation to y_pred, use output_transform as shown below:
 
-    .. code-block:: python
+    .. testcode::
 
-        def activated_output_transform(output):
-            y_pred, y = output
-            y_pred = torch.sigmoid(y_pred)
-            return y_pred, y
+        roc_auc = RocCurve(sigmoid_output_transform)
+        roc_auc.attach(default_evaluator, 'roc_auc')
+        y_pred = torch.tensor([-3, 0.4, 0.9, 8])
+        y_true = torch.tensor([0, 0, 1, 0])
+        state = default_evaluator.run([[y_pred, y_true]])
+        print("FPR", [round(i, 3) for i in state.metrics['roc_auc'][0].tolist()])
+        print("TPR", [round(i, 3) for i in state.metrics['roc_auc'][1].tolist()])
+        print("Thresholds", [round(i, 3) for i in state.metrics['roc_auc'][2].tolist()])
 
-        roc_auc = RocCurve(activated_output_transform)
+    .. testoutput::
 
+        FPR [0.0, 0.333, 0.333, 1.0]
+        TPR [0.0, 0.0, 1.0, 1.0]
+        Thresholds [2.0, 1.0, 0.711, 0.047]
     """
 
     def __init__(self, output_transform: Callable = lambda x: x, check_compute_fn: bool = False) -> None: