chapter09_part01_image-segmentation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "This is a companion notebook for the book [Deep Learning with Python, Second Edition](https://www.manning.com/books/deep-learning-with-python-second-edition?a_aid=keras&a_bid=76564dff). For readability, it only contains runnable code blocks and section titles, and omits everything else in the book: text paragraphs, figures, and pseudocode.\n\n**If you want to be able to follow what's going on, I recommend reading the notebook side by side with your copy of the book.**\n\nThis notebook was generated for TensorFlow 2.6."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "# Advanced deep learning for computer vision"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Three essential computer vision tasks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## An image segmentation example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "!wget http://www.robots.ox.ac.uk/~vgg/data/pets/data/images.tar.gz\n",
    "!wget http://www.robots.ox.ac.uk/~vgg/data/pets/data/annotations.tar.gz\n",
    "!tar -xf images.tar.gz\n",
    "!tar -xf annotations.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "input_dir = \"images/\"\n",
    "target_dir = \"annotations/trimaps/\"\n",
    "\n",
    "input_img_paths = sorted(\n",
    "    [os.path.join(input_dir, fname)\n",
    "     for fname in os.listdir(input_dir)\n",
    "     if fname.endswith(\".jpg\")])\n",
    "target_paths = sorted(\n",
    "    [os.path.join(target_dir, fname)\n",
    "     for fname in os.listdir(target_dir)\n",
    "     if fname.endswith(\".png\") and not fname.startswith(\".\")])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from tensorflow.keras.utils import load_img, img_to_array\n",
    "\n",
    "plt.axis(\"off\")\n",
    "plt.imshow(load_img(input_img_paths[9]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "def display_target(target_array):\n",
    "    normalized_array = (target_array.astype(\"uint8\") - 1) * 127\n",
    "    plt.axis(\"off\")\n",
    "    plt.imshow(normalized_array[:, :, 0])\n",
    "\n",
    "img = img_to_array(load_img(target_paths[9], color_mode=\"grayscale\"))\n",
    "display_target(img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import random\n",
    "\n",
    "img_size = (200, 200)\n",
    "num_imgs = len(input_img_paths)\n",
    "\n",
    "random.Random(1337).shuffle(input_img_paths)\n",
    "random.Random(1337).shuffle(target_paths)\n",
    "\n",
    "def path_to_input_image(path):\n",
    "    return img_to_array(load_img(path, target_size=img_size))\n",
    "\n",
    "def path_to_target(path):\n",
    "    img = img_to_array(\n",
    "        load_img(path, target_size=img_size, color_mode=\"grayscale\"))\n",
    "    img = img.astype(\"uint8\") - 1\n",
    "    return img\n",
    "\n",
    "input_imgs = np.zeros((num_imgs,) + img_size + (3,), dtype=\"float32\")\n",
    "targets = np.zeros((num_imgs,) + img_size + (1,), dtype=\"uint8\")\n",
    "for i in range(num_imgs):\n",
    "    input_imgs[i] = path_to_input_image(input_img_paths[i])\n",
    "    targets[i] = path_to_target(target_paths[i])\n",
    "\n",
    "num_val_samples = 1000\n",
    "train_input_imgs = input_imgs[:-num_val_samples]\n",
    "train_targets = targets[:-num_val_samples]\n",
    "val_input_imgs = input_imgs[-num_val_samples:]\n",
    "val_targets = targets[-num_val_samples:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "\n",
    "def get_model(img_size, num_classes):\n",
    "    inputs = keras.Input(shape=img_size + (3,))\n",
    "    x = layers.Rescaling(1./255)(inputs)\n",
    "\n",
    "    x = layers.Conv2D(64, 3, strides=2, activation=\"relu\", padding=\"same\")(x)\n",
    "    x = layers.Conv2D(64, 3, activation=\"relu\", padding=\"same\")(x)\n",
    "    x = layers.Conv2D(128, 3, strides=2, activation=\"relu\", padding=\"same\")(x)\n",
    "    x = layers.Conv2D(128, 3, activation=\"relu\", padding=\"same\")(x)\n",
    "    x = layers.Conv2D(256, 3, strides=2, padding=\"same\", activation=\"relu\")(x)\n",
    "    x = layers.Conv2D(256, 3, activation=\"relu\", padding=\"same\")(x)\n",
    "\n",
    "    x = layers.Conv2DTranspose(256, 3, activation=\"relu\", padding=\"same\")(x)\n",
    "    x = layers.Conv2DTranspose(256, 3, activation=\"relu\", padding=\"same\", strides=2)(x)\n",
    "    x = layers.Conv2DTranspose(128, 3, activation=\"relu\", padding=\"same\")(x)\n",
    "    x = layers.Conv2DTranspose(128, 3, activation=\"relu\", padding=\"same\", strides=2)(x)\n",
    "    x = layers.Conv2DTranspose(64, 3, activation=\"relu\", padding=\"same\")(x)\n",
    "    x = layers.Conv2DTranspose(64, 3, activation=\"relu\", padding=\"same\", strides=2)(x)\n",
    "\n",
    "    outputs = layers.Conv2D(num_classes, 3, activation=\"softmax\", padding=\"same\")(x)\n",
    "\n",
    "    model = keras.Model(inputs, outputs)\n",
    "    return model\n",
    "\n",
    "model = get_model(img_size=img_size, num_classes=3)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "model.compile(optimizer=\"rmsprop\", loss=\"sparse_categorical_crossentropy\")\n",
    "\n",
    "callbacks = [\n",
    "    keras.callbacks.ModelCheckpoint(\"oxford_segmentation.keras\",\n",
    "                                    save_best_only=True)\n",
    "]\n",
    "\n",
    "history = model.fit(train_input_imgs, train_targets,\n",
    "                    epochs=50,\n",
    "                    callbacks=callbacks,\n",
    "                    batch_size=64,\n",
    "                    validation_data=(val_input_imgs, val_targets))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "epochs = range(1, len(history.history[\"loss\"]) + 1)\n",
    "loss = history.history[\"loss\"]\n",
    "val_loss = history.history[\"val_loss\"]\n",
    "plt.figure()\n",
    "plt.plot(epochs, loss, \"bo\", label=\"Training loss\")\n",
    "plt.plot(epochs, val_loss, \"b\", label=\"Validation loss\")\n",
    "plt.title(\"Training and validation loss\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.utils import array_to_img\n",
    "\n",
    "model = keras.models.load_model(\"oxford_segmentation.keras\")\n",
    "\n",
    "i = 4\n",
    "test_image = val_input_imgs[i]\n",
    "plt.axis(\"off\")\n",
    "plt.imshow(array_to_img(test_image))\n",
    "\n",
    "mask = model.predict(np.expand_dims(test_image, 0))[0]\n",
    "\n",
    "def display_mask(pred):\n",
    "    mask = np.argmax(pred, axis=-1)\n",
    "    mask *= 127\n",
    "    plt.axis(\"off\")\n",
    "    plt.imshow(mask)\n",
    "\n",
    "display_mask(mask)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "chapter09_part01_image-segmentation.i",
   "private_outputs": false,
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}