hw1_problem3.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "hw1_problem3.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/PriyankaGona/MachineLearning-Assignments/blob/master/hw1_problem3.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "izNUruY_GmTo",
        "colab_type": "code",
        "outputId": "0e318b3d-5247-4d3e-b432-88f0f71ccdb3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "import keras\n",
        "import numpy as np\n",
        "from keras.datasets import mnist\n",
        "import random"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "1g1KclWrGx4z",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# convert values to binary values\n",
        "def to_categorical(y,nb_classes):\n",
        "  output = []\n",
        "  for i in y:\n",
        "    temp = np.zeros(nb_classes)\n",
        "    temp[i] = 1\n",
        "    output.append(temp)\n",
        "  output = np.array(output)\n",
        "  output = output.reshape(y.shape[0],nb_classes)\n",
        "  return output"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "dAMndbX-Gy8t",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# initialize weights and bias\n",
        "def init(dim):\n",
        "  W = np.random.randn(nb_classes,dim)*0.01\n",
        "  b = np.random.randn(nb_classes,1)\n",
        "  return W,b"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "-B9b3vdQG2Nc",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# softmax function\n",
        "def softmax(X):\n",
        "    return np.exp(X)/np.sum(np.exp(X), axis = 0, keepdims = True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "fqxFP-pTG88I",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# forward propagation\n",
        "def forward_propagate(W,b,x):\n",
        "  linear_transformation = np.dot(W,np.transpose(x)) + (b)\n",
        "  output = softmax(linear_transformation)\n",
        "  return output"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "rTHvXEFTHBFm",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# crossentropy loss function\n",
        "def crossentropy_loss(y_train,output,batch_size):\n",
        "  loss = -np.sum(np.multiply(np.transpose(y_train),np.log(output))+np.multiply(np.transpose(y_train),np.log(output)),axis=1,keepdims=True)/batch_size\n",
        "  return np.squeeze(loss)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "lAr0RFXyHDiz",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# backward propogation\n",
        "def backward_propagate(x_train,y_train,output,W,b,learning_rate,batch_size):\n",
        "  dz = output - np.transpose(y_train)\n",
        "  dW = np.dot(dz,x_train)\n",
        "  db = np.sum(dz,axis=1,keepdims=True)\n",
        "  W = W - learning_rate*dW\n",
        "  b = b - learning_rate*db\n",
        "  return W,b"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "2s2GT01LlPhN",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# shuffle data in input and output respectively\n",
        "def shuffle_list(*ls):\n",
        "    l=list(zip(*ls))\n",
        "    random.shuffle(l)\n",
        "    return zip(*l)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "t_u95Y3jHIY2",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# model using mini-batch stochastic gradient descent\n",
        "def model(W,b,x_train,y_train,learning_rate,batch_size,epochs):\n",
        "  for j in range(epochs):\n",
        "    training_loss = []\n",
        "\n",
        "    x_train, y_train = shuffle_list(x_train, y_train)\n",
        "    x_train=np.asarray(x_train)\n",
        "    y_train=np.asarray(y_train)\n",
        "    \n",
        "    for i in range(0, x_train.shape[0], batch_size):\n",
        "      x_train_mini = x_train[i:i+batch_size]\n",
        "      y_train_mini = y_train[i:i+batch_size]\n",
        "      \n",
        "      output = forward_propagate(W,b,x_train_mini)\n",
        "      \n",
        "      loss_matrix = crossentropy_loss(y_train_mini,output,batch_size)\n",
        "      loss_list = loss_matrix.tolist()\n",
        "      training_loss.extend(loss_list)\n",
        "      \n",
        "      W,b = backward_propagate(x_train_mini,y_train_mini,output,W,b,learning_rate,batch_size)\n",
        "    \n",
        "    epochLoss=sum(training_loss)/len(training_loss)\n",
        "    print(\"Epoch {}/{}\\t - loss : {}\".format(j+1,epochs,round(epochLoss,4)))\n",
        "\n",
        "  return W,b"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "WMNxKP7KHLaL",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# calculating accuracy of test data after training  \n",
        "def predict(W, b, X, Y):\n",
        "    Y_prediction = forward_propagate(W,b,X)\n",
        "    Y_prediction = Y_prediction.T\n",
        "    accuracy = 100 - np.mean(np.abs(Y_prediction - Y)) * 100\n",
        "    return round(accuracy,4)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "NpG_y4lmHOEx",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# variables initialization\n",
        "learning_rate = 0.005\n",
        "batch_size = 8\n",
        "experiments = 20\n",
        "nb_classes=10"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "UbNOuDT7HRwj",
        "colab_type": "code",
        "outputId": "0c1b4165-eec3-4820-a08a-9159b562da83",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 374
        }
      },
      "cell_type": "code",
      "source": [
        "# mnist dataset loading using keras\n",
        "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
        "\n",
        "# reshaping the image to 1-dimensional vector\n",
        "x_train = x_train.reshape(x_train.shape[0],x_train.shape[1]*x_train.shape[2])\n",
        "x_test = x_test.reshape(x_test.shape[0],x_test.shape[1]*x_test.shape[2])\n",
        "y_actual_testdata = y_test\n",
        "\n",
        "y_train = to_categorical(y_train,nb_classes)\n",
        "y_test = to_categorical(y_test,nb_classes)\n",
        "\n",
        "x_train = x_train.astype('float32')\n",
        "x_test = x_test.astype('float32')\n",
        "\n",
        "# normalizing\n",
        "x_train /= 255\n",
        "x_test /= 255\n",
        "\n",
        "\n",
        "W,b = init(x_train.shape[1])\n",
        "W,b = model(W,b,x_train,y_train,learning_rate,batch_size,experiments)\n",
        "\n",
        "acc = predict(W,b,x_test,y_test)\n",
        "print(\"Accuracy: {}\".format(acc))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/20\t - loss : 0.0766\n",
            "Epoch 2/20\t - loss : 0.0609\n",
            "Epoch 3/20\t - loss : 0.058\n",
            "Epoch 4/20\t - loss : 0.0566\n",
            "Epoch 5/20\t - loss : 0.0556\n",
            "Epoch 6/20\t - loss : 0.0547\n",
            "Epoch 7/20\t - loss : 0.0542\n",
            "Epoch 8/20\t - loss : 0.0537\n",
            "Epoch 9/20\t - loss : 0.0534\n",
            "Epoch 10/20\t - loss : 0.0528\n",
            "Epoch 11/20\t - loss : 0.0526\n",
            "Epoch 12/20\t - loss : 0.0525\n",
            "Epoch 13/20\t - loss : 0.0521\n",
            "Epoch 14/20\t - loss : 0.0519\n",
            "Epoch 15/20\t - loss : 0.0518\n",
            "Epoch 16/20\t - loss : 0.0515\n",
            "Epoch 17/20\t - loss : 0.0514\n",
            "Epoch 18/20\t - loss : 0.0511\n",
            "Epoch 19/20\t - loss : 0.051\n",
            "Epoch 20/20\t - loss : 0.0508\n",
            "Accuracy: 97.6619\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}