Tensorflow - Text Classification using Tensorflow Hub.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "view-in-github"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/sahug/ds-tensorflow-colab/blob/master/Tensorflow%20-%20Text%20Classification%20using%20Tensorflow%20Hub.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KGRJYY4jnldA"
      },
      "source": [
        "**Tensorflow - Text Classification using Tensorflow Hub**\n",
        "\n",
        "The tutorial demonstrates the basic application of transfer learning with TensorFlow Hub and Keras.\n",
        "\n",
        "In transfer learning we use the knowledge from the previous model to build next model. In here we will use embedding from hub to build a Text Classification Model."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1Gjroa3undk_",
        "outputId": "3a8900f1-8715-41a5-bbbf-11c2ba8b256d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Note: you may need to restart the kernel to use updated packages.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING: You are using pip version 22.0.4; however, version 22.1.1 is available.\n",
            "You should consider upgrading via the 'c:\\Users\\sahug\\AppData\\Local\\Programs\\Python\\Python310\\python.exe -m pip install --upgrade pip' command.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Note: you may need to restart the kernel to use updated packages.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING: You are using pip version 22.0.4; however, version 22.1.1 is available.\n",
            "You should consider upgrading via the 'c:\\Users\\sahug\\AppData\\Local\\Programs\\Python\\Python310\\python.exe -m pip install --upgrade pip' command.\n"
          ]
        }
      ],
      "source": [
        "%pip install -q tensorflow-hub\n",
        "%pip install -q tensorflow-datasets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "51yBqf9MoF60"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\sahug\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\tqdm\\auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
            "  from .autonotebook import tqdm as notebook_tqdm\n"
          ]
        }
      ],
      "source": [
        "import os\n",
        "import numpy as np\n",
        "\n",
        "import tensorflow as tf\n",
        "import tensorflow_hub as hub\n",
        "import tensorflow_datasets as tfds"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Mst78LqRoUNv",
        "outputId": "759d7e78-e221-48bb-8bf7-e81e40496ac9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Version:  2.8.0\n",
            "Eager mode:  True\n",
            "Hub version:  0.12.0\n",
            "GPU is NOT AVAILABLE\n"
          ]
        }
      ],
      "source": [
        "print(\"Version: \", tf.__version__)\n",
        "print(\"Eager mode: \", tf.executing_eagerly())\n",
        "print(\"Hub version: \", hub.__version__)\n",
        "print(\"GPU is\", \"available\" if tf.config.list_physical_devices(\"GPU\") else \"NOT AVAILABLE\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l0liOnjPoYxF"
      },
      "source": [
        "**Download and Exlpore Data**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "04C2ahb0opBU"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[1mDownloading and preparing dataset Unknown size (download: Unknown size, generated: Unknown size, total: Unknown size) to C:\\Users\\sahug\\tensorflow_datasets\\imdb_reviews\\plain_text\\1.0.0...\u001b[0m\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Dl Size...: 100%|██████████| 80/80 [00:37<00:00,  2.11 MiB/s]rl]\n",
            "Dl Completed...: 100%|██████████| 1/1 [00:37<00:00, 37.99s/ url]\n",
            "                                                                        \r"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[1mDataset imdb_reviews downloaded and prepared to C:\\Users\\sahug\\tensorflow_datasets\\imdb_reviews\\plain_text\\1.0.0. Subsequent calls will reuse this data.\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "# Split the training set into 60% and 40% to end up with 15,000 examples\n",
        "# for training, 10,000 examples for validation and 25,000 examples for testing.\n",
        "train_data, validation_data, test_data = tfds.load(\n",
        "    name=\"imdb_reviews\",\n",
        "    split=(\"train[:60%]\", \"train[60%:]\", \"test\"),\n",
        "    as_supervised=True\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mb4-m6UMoWyl",
        "outputId": "e7386ff3-91bb-49f3-bac0-a7551231a2aa"
      },
      "outputs": [],
      "source": [
        "train_example_batch, train_labels_batch =  next(iter(train_data.batch(10)))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y3AxEtqipcnT"
      },
      "source": [
        "**Build the Model**\n",
        "\n",
        "The neural network is created by stacking layers—this requires three main architectural decisions:\n",
        "\n",
        "- How to represent the text?\n",
        "- How many layers to use in the model?\n",
        "- How many hidden units to use for each layer?"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HEbEnkrUpm0k"
      },
      "source": [
        "Let's first create a **Keras** layer that uses a **TensorFlow Hub model** to embed the sentences, and try it out on a couple of input examples. Note that no matter the length of the input text, the output shape of the embeddings is: **(num_examples, embedding_dimension)**."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F30ZOb79r0CU"
      },
      "source": [
        "**Embedding**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-jSHz1MypeGW",
        "outputId": "33793a6c-c99e-4326-c9e8-1de5f8c5376d"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(2, 50), dtype=float32, numpy=\n",
              "array([[ 0.5423195 , -0.0119017 ,  0.06337538,  0.06862972, -0.16776837,\n",
              "        -0.10581174,  0.16865303, -0.04998824, -0.31148055,  0.07910346,\n",
              "         0.15442263,  0.01488662,  0.03930153,  0.19772711, -0.12215476,\n",
              "        -0.04120981, -0.2704109 , -0.21922152,  0.26517662, -0.80739075,\n",
              "         0.25833532, -0.3100421 ,  0.28683215,  0.1943387 , -0.29036492,\n",
              "         0.03862849, -0.7844411 , -0.0479324 ,  0.4110299 , -0.36388892,\n",
              "        -0.58034706,  0.30269456,  0.3630897 , -0.15227164, -0.44391504,\n",
              "         0.19462997,  0.19528408,  0.05666234,  0.2890704 , -0.28468323,\n",
              "        -0.00531206,  0.0571938 , -0.3201318 , -0.04418665, -0.08550783,\n",
              "        -0.55847436, -0.23336391, -0.20782952, -0.03543064, -0.17533456],\n",
              "       [ 0.56338924, -0.12339553, -0.10862679,  0.7753425 , -0.07667089,\n",
              "        -0.15752277,  0.01872335, -0.08169781, -0.3521876 ,  0.4637341 ,\n",
              "        -0.08492756,  0.07166859, -0.00670817,  0.12686075, -0.19326553,\n",
              "        -0.52626437, -0.3295823 ,  0.14394785,  0.09043556, -0.5417555 ,\n",
              "         0.02468163, -0.15456742,  0.68333143,  0.09068331, -0.45327246,\n",
              "         0.23180096, -0.8615696 ,  0.34480393,  0.12838456, -0.58759046,\n",
              "        -0.4071231 ,  0.23061076,  0.48426893, -0.27128142, -0.5380916 ,\n",
              "         0.47016326,  0.22572741, -0.00830663,  0.2846242 , -0.304985  ,\n",
              "         0.04400365,  0.25025874,  0.14867121,  0.40717036, -0.15422426,\n",
              "        -0.06878027, -0.40825695, -0.3149215 ,  0.09283665, -0.20183425]],\n",
              "      dtype=float32)>"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "embedding = \"https://tfhub.dev/google/nnlm-en-dim50/2\"\n",
        "hub_layer = hub.KerasLayer(embedding, input_shape=[], dtype=tf.string, trainable=True)\n",
        "hub_layer(train_example_batch[:2])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "saSEfayFr1zQ"
      },
      "source": [
        "**Model**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3Ixp1-IYq3Gv",
        "outputId": "17a01061-1061-43ef-dbe9-2e76085951b8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " keras_layer (KerasLayer)    (None, 50)                48190600  \n",
            "                                                                 \n",
            " dense (Dense)               (None, 16)                816       \n",
            "                                                                 \n",
            " dense_1 (Dense)             (None, 1)                 17        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 48,191,433\n",
            "Trainable params: 48,191,433\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "from keras import activations\n",
        "model = tf.keras.Sequential()\n",
        "model.add(hub_layer)\n",
        "model.add(tf.keras.layers.Dense(16, activation=activations.relu))\n",
        "model.add(tf.keras.layers.Dense(1))\n",
        "model.summary()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-kJyvvU_r3Wa"
      },
      "source": [
        "**Loss Function and Optimizer**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "gQjrWgS5r54p"
      },
      "outputs": [],
      "source": [
        "from tensorflow import keras\n",
        "model.compile(optimizer=tf.keras.optimizers.Adam(), \n",
        "                loss=keras.losses.BinaryCrossentropy(),\n",
        "                metrics=[\"accuracy\"])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ANRHJelPsGIa"
      },
      "source": [
        "**Train**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jxfEdfUfsHa9",
        "outputId": "bebe15e0-48a8-4ece-9765-265e0d410f12"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/10\n",
            "30/30 [==============================] - 16s 524ms/step - loss: 7.7007 - accuracy: 0.5007 - val_loss: 7.7215 - val_accuracy: 0.4990\n",
            "Epoch 2/10\n",
            "30/30 [==============================] - 16s 548ms/step - loss: 4.2408 - accuracy: 0.5193 - val_loss: 1.2195 - val_accuracy: 0.5504\n",
            "Epoch 3/10\n",
            "30/30 [==============================] - 17s 574ms/step - loss: 0.9749 - accuracy: 0.6108 - val_loss: 0.8371 - val_accuracy: 0.6292\n",
            "Epoch 4/10\n",
            "30/30 [==============================] - 17s 575ms/step - loss: 0.6473 - accuracy: 0.7063 - val_loss: 0.6892 - val_accuracy: 0.6900\n",
            "Epoch 5/10\n",
            "30/30 [==============================] - 16s 547ms/step - loss: 0.5093 - accuracy: 0.7915 - val_loss: 0.6214 - val_accuracy: 0.7330\n",
            "Epoch 6/10\n",
            "30/30 [==============================] - 17s 562ms/step - loss: 0.4257 - accuracy: 0.8451 - val_loss: 0.5882 - val_accuracy: 0.7653\n",
            "Epoch 7/10\n",
            "30/30 [==============================] - 16s 535ms/step - loss: 0.3514 - accuracy: 0.8809 - val_loss: 0.5607 - val_accuracy: 0.7878\n",
            "Epoch 8/10\n",
            "30/30 [==============================] - 17s 566ms/step - loss: 0.2892 - accuracy: 0.9089 - val_loss: 0.5508 - val_accuracy: 0.8074\n",
            "Epoch 9/10\n",
            "30/30 [==============================] - 20s 684ms/step - loss: 0.2413 - accuracy: 0.9288 - val_loss: 0.5484 - val_accuracy: 0.8167\n",
            "Epoch 10/10\n",
            "30/30 [==============================] - 22s 735ms/step - loss: 0.2026 - accuracy: 0.9418 - val_loss: 0.5392 - val_accuracy: 0.8250\n"
          ]
        }
      ],
      "source": [
        "history = model.fit(train_data.shuffle(10000).batch(512), \n",
        "                    epochs=10, \n",
        "                    validation_data=validation_data.batch(512), \n",
        "                    verbose=1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hpBUME1ltM0h"
      },
      "source": [
        "**Evaluate**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JMpC9ivQtOVf",
        "outputId": "91a9e3a2-f7a7-42a4-a16a-6d71155750c9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "49/49 - 3s - loss: 0.5808 - accuracy: 0.8055 - 3s/epoch - 67ms/step\n",
            "loss: 0.581\n",
            "accuracy: 0.805\n"
          ]
        }
      ],
      "source": [
        "result = model.evaluate(test_data.batch(512), verbose=2)\n",
        "for name, value in zip(model.metrics_names, result):\n",
        "  print(\"%s: %.3f\" % (name, value))"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "authorship_tag": "ABX9TyPF6aQNCk2NyST+VwL4Mrth",
      "collapsed_sections": [],
      "include_colab_link": true,
      "name": "Tensorflow - Text Classification using Tensorflow Hub.ipynb",
      "provenance": []
    },
    "interpreter": {
      "hash": "0d7fd96aeadd740978789f2880dd43822d8681a39e327029c33ff72656207c4b"
    },
    "kernelspec": {
      "display_name": "Python 3.10.4 64-bit",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.4"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}