NN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\prava\\anaconda3\\envs\\pytorch\\lib\\site-packages\\tqdm\\auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from torch.utils.data import Dataset, DataLoader, random_split\n",
    "from torch.optim import lr_scheduler\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from seaborn import countplot, scatterplot, catplot\n",
    "from tqdm.auto import tqdm\n",
    "import copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    }
   ],
   "source": [
    "# Make use of GPU if available\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dataset\n",
    "class PMTDataset(Dataset):\n",
    "    def __init__(self, csv_file):\n",
    "\n",
    "        self.df = pd.read_csv(csv_file, encoding= 'unicode_escape')\n",
    "\n",
    "        # Encode the label i.e. the 'isKilled' column --> no=1 and yes=0\n",
    "        self.y = torch.tensor(pd.Series(np.where(self.df.isKilled.values == 'no', 1., 0.), self.df.index).to_numpy()).to(device)\n",
    "\n",
    "        # Encode the categorical features to one hot encodings\n",
    "        self.df['operator'] = pd.Categorical(self.df['operator'], categories=['T0','T1','T2','T3','T4','T5','T6','T7','T8','T9','T10','T11','T12','T13','T14'])\n",
    "        self.df['methodReturn'] = pd.Categorical(self.df['methodReturn'], categories=['I','V','Z','method','D','[D','[[D','J','[I','C','[J','[C','[S',\n",
    "                                                                                      'F','[F','[B','S','B','[Z','[[S','[[B','[[Z'])\n",
    "        self.X = torch.tensor(pd.get_dummies(self.df, columns=[\"operator\", \"methodReturn\"]).drop(\"isKilled\", axis=1).to_numpy()).to(device)\n",
    "        # self.X = self.X / self.X.max(0, keepdim=True)[0]\n",
    "        # self.X[torch.isnan(self.X)] = 0\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return self.X[idx], self.y[idx]\n",
    "    \n",
    "    def get_data(self):\n",
    "        return self.X, self.y\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model\n",
    "class Net(nn.Module):\n",
    "    def __init__(self, input_size, h1_size, h2_size, h3_size, h4_size, num_classes):\n",
    "        super().__init__()\n",
    "        self.l1 = nn.Linear(input_size, h1_size)\n",
    "        self.tanh = nn.Tanh()\n",
    "        self.l2 = nn.Linear(h1_size, h2_size)\n",
    "        # self.l3 = nn.Linear(h2_size, h3_size)\n",
    "        # self.l4 = nn.Linear(h3_size, h4_size)\n",
    "        self.l5 = nn.Linear(h2_size, num_classes)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        out = self.l1(x.float())\n",
    "        out = self.tanh(out)\n",
    "        out = self.l2(out)\n",
    "        out = self.tanh(out)\n",
    "        # out = self.l3(out)\n",
    "        # out = self.tanh(out)\n",
    "        # out = self.l4(out)\n",
    "        # out = self.tanh(out)\n",
    "        out = self.l5(out)\n",
    "        return out\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyperparameters\n",
    "input_size = 49\n",
    "h1_size = 50\n",
    "h2_size = 10\n",
    "h3_size = 16\n",
    "h4_size = 8\n",
    "num_classes = 1\n",
    "num_epochs = 5\n",
    "num_batches = 500\n",
    "learning_rate = 0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1184, 49]) torch.Size([1184])\n"
     ]
    }
   ],
   "source": [
    "train_dataset = PMTDataset(\"training_data.csv\")\n",
    "train_loader = DataLoader(dataset=train_dataset, batch_size=len(train_dataset)//num_batches +1, shuffle=True)\n",
    "test_dataset = PMTDataset(\"testing_data.csv\")\n",
    "# test_loader = DataLoader(dataset=train_dataset, batch_size=len(train_dataset), shuffle=False)\n",
    "\n",
    "examples = iter(train_loader)\n",
    "samples, labels = examples.next()\n",
    "print(samples.shape, labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Net(input_size, h1_size, h2_size, h3_size, h4_size, num_classes).to(device)\n",
    "\n",
    "#loss and optimizer\n",
    "criterion = nn.BCEWithLogitsLoss(pos_weight=torch.FloatTensor([3.866]).to(device))\n",
    "# criterion = nn.BCEWithLogitsLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr = learning_rate)\n",
    "\n",
    "# LR scheduler\n",
    "lambda1 = lambda epoch: 0.5\n",
    "scheduler = lr_scheduler.MultiplicativeLR(optimizer, lambda1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def confusion(truth, prediction):\n",
    "    \"\"\" Returns the confusion matrix for the values in the `prediction` and `truth`\n",
    "    tensors, i.e. the amount of positions where the values of `prediction`\n",
    "    and `truth` are\n",
    "    - 1 and 1 (True Positive)\n",
    "    - 1 and 0 (False Positive)\n",
    "    - 0 and 0 (True Negative)\n",
    "    - 0 and 1 (False Negative)\n",
    "    \"\"\"\n",
    "\n",
    "    confusion_vector = prediction / truth\n",
    "    # Element-wise division of the 2 tensors returns a new tensor which holds a\n",
    "    # unique value for each case:\n",
    "    #   1     where prediction and truth are 1 (True Positive)\n",
    "    #   inf   where prediction is 1 and truth is 0 (False Positive)\n",
    "    #   nan   where prediction and truth are 0 (True Negative)\n",
    "    #   0     where prediction is 0 and truth is 1 (False Negative)\n",
    "\n",
    "    true_positives = torch.sum(confusion_vector == 1).item()\n",
    "    false_positives = torch.sum(confusion_vector == float('inf')).item()\n",
    "    true_negatives = torch.sum(torch.isnan(confusion_vector)).item()\n",
    "    false_negatives = torch.sum(confusion_vector == 0).item()\n",
    "\n",
    "    print(true_positives, false_negatives)\n",
    "    print(false_positives, true_negatives)\n",
    "    # return [[true_positives, false_positives], [false_negatives, true_negatives]]\n",
    "\n",
    "# testing loop\n",
    "def test():\n",
    "    with torch.no_grad():\n",
    "        n_correct = 0\n",
    "\n",
    "        features, labels = test_dataset.get_data()\n",
    "\n",
    "        outputs = model(features)\n",
    "\n",
    "        predictions = torch.round(torch.sigmoid(outputs)).view(-1)\n",
    "        # print(predictions.shape, labels.shape, type(labels))\n",
    "        \n",
    "        n_correct = torch.sum(predictions == labels).item()\n",
    "        # confusion(labels, predictions)\n",
    "\n",
    "        acc = 100.0 * n_correct / len(labels)\n",
    "        # print('accuracy = %.3f' % acc)\n",
    "        return acc\n",
    "\n",
    "# testing loop\n",
    "def train():\n",
    "    with torch.no_grad():\n",
    "        n_correct = 0\n",
    "\n",
    "        features, labels = train_dataset.get_data()\n",
    "\n",
    "        outputs = model(features)\n",
    "\n",
    "        predictions = torch.round(torch.sigmoid(outputs)).view(-1)\n",
    "        # print(predictions.shape, labels.shape, type(labels))\n",
    "        \n",
    "        n_correct = torch.sum(predictions == labels).item()\n",
    "        # confusion(labels, predictions)\n",
    "\n",
    "        acc = 100.0 * n_correct / len(labels)\n",
    "        # print('accuracy = %.3f' % acc)\n",
    "        return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loss = 0.1262, train accuracy = 94.426, test accuracy = 86.063, best accuracy = 89.401: 100%|██████████| 500/500 [00:08<00:00, 57.94it/s]\n",
      "loss = 0.1312, train accuracy = 94.455, test accuracy = 86.098, best accuracy = 89.449: 100%|██████████| 500/500 [00:08<00:00, 58.84it/s]\n",
      "loss = 0.1200, train accuracy = 94.448, test accuracy = 88.835, best accuracy = 89.449: 100%|██████████| 500/500 [00:08<00:00, 58.69it/s]\n",
      "loss = 0.1573, train accuracy = 94.503, test accuracy = 88.444, best accuracy = 89.449: 100%|██████████| 500/500 [00:08<00:00, 58.95it/s]\n",
      "loss = 0.1122, train accuracy = 94.512, test accuracy = 88.535, best accuracy = 89.449: 100%|██████████| 500/500 [00:08<00:00, 58.41it/s]\n"
     ]
    }
   ],
   "source": [
    "#training loop\n",
    "total_steps = len(train_loader)\n",
    "best_acc = 0.0\n",
    "best_model = 0\n",
    "for epoch in range(num_epochs):\n",
    "    pbar = tqdm(train_loader)\n",
    "    for (features, labels) in pbar:\n",
    "        # print(features.dtype, labels.dtype)\n",
    "        features = features.to(device)\n",
    "        labels = labels.to(device)\n",
    "\n",
    "        #forward\n",
    "        outputs = model(features)\n",
    "        #print(outputs.dtype)\n",
    "        loss = criterion(outputs, labels.unsqueeze(1))\n",
    "\n",
    "        #backward\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        test_acc = test()\n",
    "        if best_acc < test_acc:\n",
    "            best_acc = test_acc\n",
    "            best_model = copy.deepcopy(model)\n",
    "\n",
    "        pbar.set_description(f'loss = {loss.item():.4f}, train accuracy = {train():.3f}, test accuracy = {test_acc:.3f}, best accuracy = {best_acc:.3f}')\n",
    "        #if (i+1) % 100 == 0:\n",
    "        # print(f'epoch {epoch+1} / {num_epochs}, step {i+1}/{total_steps}, loss = {loss.item():.4f}')\n",
    "    scheduler.step()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "456128 14062\n",
      "37069 84554\n"
     ]
    }
   ],
   "source": [
    "# Train Data\n",
    "\n",
    "n_correct = 0\n",
    "\n",
    "features, labels = train_dataset.get_data()\n",
    "\n",
    "outputs = model(features)\n",
    "\n",
    "predictions = torch.round(torch.sigmoid(outputs)).view(-1)\n",
    "# print(predictions.shape, labels.shape, type(labels))\n",
    "\n",
    "n_correct = torch.sum(predictions == labels).item()\n",
    "confusion(labels, predictions)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "87372 5610\n",
      "22230 33280\n"
     ]
    }
   ],
   "source": [
    "# Test Data\n",
    "\n",
    "n_correct = 0\n",
    "\n",
    "features, labels = test_dataset.get_data()\n",
    "\n",
    "outputs = model(features)\n",
    "\n",
    "predictions = torch.round(torch.sigmoid(outputs)).view(-1)\n",
    "# print(predictions.shape, labels.shape, type(labels))\n",
    "\n",
    "n_correct = torch.sum(predictions == labels).item()\n",
    "confusion(labels, predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "a9377cdb45579e7d070d08896ef673c50b9baa4fee8da92d370a91ef6fc257de"
  },
  "kernelspec": {
   "display_name": "Python 3.8.12 ('pytorch')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}