unet4nuclei/04-evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 04\n",
    "# Evaluation of performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sb\n",
    "\n",
    "import skimage.io\n",
    "import skimage.morphology\n",
    "import skimage.segmentation\n",
    "\n",
    "import utils.evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from config import config_vars\n",
    "\n",
    "# Partition of the data to make predictions (test or validation)\n",
    "partition = \"validation\"\n",
    "\n",
    "experiment_name = '01'\n",
    "\n",
    "config_vars = utils.dirtools.setup_experiment(config_vars, experiment_name)\n",
    "\n",
    "data_partitions = utils.dirtools.read_data_partitions(config_vars)\n",
    "\n",
    "config_vars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To evaluate segmentations produced by other segmentation algorithms (e.g. CellProfiler), \n",
    "# manually modify the following config var:\n",
    "\n",
    "# config_vars[\"labels_out_dir\"] = \"/data1/image-segmentation/cp3_output_2018_02_02/mask/\"\n",
    "# config_vars[\"object_dilation\"] = 3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Auxiliary visualization function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Display prediction along with segmentation to visualize errors\n",
    "\n",
    "def show(ground_truth, prediction, threshold=0.5, image_name=\"N\"):\n",
    "    \n",
    "    # Compute Intersection over Union\n",
    "    IOU = utils.evaluation.intersection_over_union(ground_truth, prediction)\n",
    "    \n",
    "    # Create diff map\n",
    "    diff = np.zeros(ground_truth.shape + (3,))\n",
    "    A = ground_truth.copy()\n",
    "    B = prediction.copy()\n",
    "    A[A > 0] = 1\n",
    "    B[B > 0] = 1\n",
    "    D = A - B\n",
    "    \n",
    "    # Object-level errors\n",
    "    C = IOU.copy()\n",
    "    C[C>=threshold] = 1\n",
    "    C[C<threshold] = 0\n",
    "    missed = np.where(np.sum(C,axis=1) == 0)[0]\n",
    "    extra = np.where(np.sum(C,axis=0) == 0)[0]\n",
    "\n",
    "    for m in missed:\n",
    "        diff[ground_truth == m+1, 0] = 1\n",
    "    for e in extra:\n",
    "        diff[prediction == e+1, 2] = 1\n",
    "    \n",
    "    # Display figures\n",
    "    fig, ax = plt.subplots(1, 4, figsize=(18,6))\n",
    "    ax[0].imshow(ground_truth)\n",
    "    ax[0].set_title(\"True objects:\"+str(len(np.unique(ground_truth))))\n",
    "    ax[1].imshow(diff)\n",
    "    ax[1].set_title(\"Segmentation errors:\"+str(len(missed)))\n",
    "    ax[2].imshow(prediction)\n",
    "    ax[2].set_title(\"Predicted objects:\"+str(len(np.unique(prediction))))\n",
    "    ax[3].imshow(IOU)\n",
    "    ax[3].set_title(image_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run the evaluation\n",
    "Predictions are stored in Step 03 and loaded here for evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "all_images = data_partitions[partition]\n",
    "\n",
    "results = pd.DataFrame(columns=[\"Image\", \"Threshold\", \"F1\", \"Jaccard\", \"TP\", \"FP\", \"FN\"])\n",
    "false_negatives = pd.DataFrame(columns=[\"False_Negative\", \"Area\"])\n",
    "splits_merges = pd.DataFrame(columns=[\"Image_Name\", \"Merges\", \"Splits\"])\n",
    "\n",
    "for image_name in all_images:\n",
    "    # Load ground truth data\n",
    "    img_filename = os.path.join(config_vars[\"raw_annotations_dir\"], image_name)\n",
    "    ground_truth = skimage.io.imread(img_filename)\n",
    "    if len(ground_truth.shape) == 3:\n",
    "        ground_truth = ground_truth[:,:,0]\n",
    "    \n",
    "    # Transform to label matrix\n",
    "    ground_truth = skimage.morphology.label(ground_truth)\n",
    "    \n",
    "    # Load predictions\n",
    "    pred_filename = os.path.join(config_vars[\"labels_out_dir\"], image_name)\n",
    "    prediction = skimage.io.imread(pred_filename) #.replace(\".png\",\".tiff\"))\n",
    "    \n",
    "    # Apply object dilation\n",
    "    if config_vars[\"object_dilation\"] > 0:\n",
    "        struct = skimage.morphology.square(config_vars[\"object_dilation\"])\n",
    "        prediction = skimage.morphology.dilation(prediction, struct)\n",
    "    elif config_vars[\"object_dilation\"] < 0:\n",
    "        struct = skimage.morphology.square(-config_vars[\"object_dilation\"])\n",
    "        prediction = skimage.morphology.erosion(prediction, struct)\n",
    "        \n",
    "    # Relabel objects (cut margin of 30 pixels to make a fair comparison with DeepCell)\n",
    "    ground_truth = skimage.segmentation.relabel_sequential(ground_truth)[0] #[30:-30,30:-30])[0]\n",
    "    prediction = skimage.segmentation.relabel_sequential(prediction)[0] #[30:-30,30:-30])[0]\n",
    "    \n",
    "    # Compute evaluation metrics\n",
    "    results = utils.evaluation.compute_af1_results(\n",
    "        ground_truth, \n",
    "        prediction, \n",
    "        results, \n",
    "        image_name\n",
    "    )\n",
    "    \n",
    "    false_negatives = utils.evaluation.get_false_negatives(\n",
    "        ground_truth, \n",
    "        prediction, \n",
    "        false_negatives, \n",
    "        image_name\n",
    "    )\n",
    "    \n",
    "    splits_merges = utils.evaluation.get_splits_and_merges(\n",
    "        ground_truth, \n",
    "        prediction, \n",
    "        splits_merges, \n",
    "        image_name\n",
    "    )\n",
    "    \n",
    "    # Display an example image\n",
    "    if image_name == all_images[0]:\n",
    "        show(ground_truth, prediction, image_name=image_name)\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Report of results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Display accuracy results\n",
    "\n",
    "results = results[results[\"Threshold\"] < 0.95]\n",
    "\n",
    "average_performance = results.groupby(\"Threshold\").mean().reset_index()\n",
    "\n",
    "R = results.groupby(\"Image\").mean().reset_index()\n",
    "g = sb.jointplot(data=R[R[\"F1\"] > 0.4], x=\"Jaccard\", y=\"F1\", xlim=[0.8,1], ylim=[0.6,1])\n",
    "g.fig.set_figwidth(4)\n",
    "g.fig.set_figheight(8)\n",
    "\n",
    "average_performance\n",
    "R.sort_values(by=\"F1\",ascending=False).loc[12, \"Image\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot accuracy results\n",
    "\n",
    "sb.regplot(data=average_performance, x=\"Threshold\", y=\"F1\", order=3, ci=None)\n",
    "average_performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute and print Average F1\n",
    "\n",
    "average_F1_score = average_performance[\"F1\"].mean()\n",
    "jaccard_index = average_performance[\"Jaccard\"].mean()\n",
    "print(\"Average F1 score:\", average_F1_score)\n",
    "print(\"Jaccard index:\", jaccard_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Summarize False Negatives by area\n",
    "\n",
    "false_negatives = false_negatives[false_negatives[\"False_Negative\"] == 1]\n",
    "\n",
    "false_negatives.groupby(\n",
    "    pd.cut(\n",
    "        false_negatives[\"Area\"], \n",
    "        [0,250,625,900,10000], # Area intervals\n",
    "        labels=[\"Tiny nuclei\",\"Small nuclei\",\"Normal nuclei\",\"Large nuclei\"],\n",
    "    )\n",
    ")[\"False_Negative\"].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Summarize splits and merges\n",
    "\n",
    "print(\"Splits:\",np.sum(splits_merges[\"Splits\"]))\n",
    "print(\"Merges:\",np.sum(splits_merges[\"Merges\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Report false positives\n",
    "\n",
    "print(\"Extra objects (false postives):\",results[results[\"Threshold\"].round(3) == 0.7].sum()[\"FP\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}