COVID-19_Analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "COVID-19 Tweets Sentiment analysis.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FfkWV6RhXify"
      },
      "source": [
        "## Importing Relevant Libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q0BMChUmMCYy"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from google.colab import drive\n",
        "import os\n",
        "import torch"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4MOJjkf8X36W"
      },
      "source": [
        "## Checking if GPU's available and weather there are multiple GPU's available.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hjWRR7te7NEj",
        "outputId": "b66b8d8f-d0ca-4323-bd11-e748a48082e7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "if torch.cuda.is_available():\n",
        "  device=torch.device(\"cuda\")\n",
        "  print(\"There are {} GPU's available.\".format(torch.cuda.device_count()))\n",
        "  print(\"We will use the GPU:{}\".format(torch.cuda.get_device_name(0)))\n",
        "else:\n",
        "  print(\"No Gpu available...using the CPU instead.\")\n",
        "  device=torch.device('cpu')"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 1 GPU's available.\n",
            "We will use the GPU:Tesla T4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "af9gs4ASYkbC"
      },
      "source": [
        "## Mounting Google Drive\n",
        "   We mount Google drive so that we can store serialized objects and intermediate data."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vNcK8CEoNaMi",
        "outputId": "d9991ac4-7d3c-4d56-c006-a3b5480cae7e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pJgXnP2POiYN"
      },
      "source": [
        "os.getcwd()\n",
        "os.chdir('/content/gdrive/My Drive/Colab Notebooks')"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A7mQSkZvZX8g"
      },
      "source": [
        "## Loading in the Dataset."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G4TBbU6QOo-d"
      },
      "source": [
        "path='/content/gdrive/My Drive/Colab Notebooks/sentiment140.csv'\n",
        "_data=pd.read_csv(path,encoding='ISO-8859-1',parse_dates=True)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uVgwebwVMfLT",
        "outputId": "ea499873-9abc-424e-a1c3-ea21fdc74601",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        }
      },
      "source": [
        "column_names=[\"target\",\"ids\",\"date\",\"flag\",\"user\",\"text\"]\n",
        "_data.columns=column_names\n",
        "_data.head()"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>target</th>\n",
              "      <th>ids</th>\n",
              "      <th>date</th>\n",
              "      <th>flag</th>\n",
              "      <th>user</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>1467810672</td>\n",
              "      <td>Mon Apr 06 22:19:49 PDT 2009</td>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>scotthamilton</td>\n",
              "      <td>is upset that he can't update his Facebook by ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>1467810917</td>\n",
              "      <td>Mon Apr 06 22:19:53 PDT 2009</td>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>mattycus</td>\n",
              "      <td>@Kenichan I dived many times for the ball. Man...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "      <td>1467811184</td>\n",
              "      <td>Mon Apr 06 22:19:57 PDT 2009</td>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>ElleCTF</td>\n",
              "      <td>my whole body feels itchy and like its on fire</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>1467811193</td>\n",
              "      <td>Mon Apr 06 22:19:57 PDT 2009</td>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>Karoli</td>\n",
              "      <td>@nationwideclass no, it's not behaving at all....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>1467811372</td>\n",
              "      <td>Mon Apr 06 22:20:00 PDT 2009</td>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>joy_wolf</td>\n",
              "      <td>@Kwesidei not the whole crew</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   target  ...                                               text\n",
              "0       0  ...  is upset that he can't update his Facebook by ...\n",
              "1       0  ...  @Kenichan I dived many times for the ball. Man...\n",
              "2       0  ...    my whole body feels itchy and like its on fire \n",
              "3       0  ...  @nationwideclass no, it's not behaving at all....\n",
              "4       0  ...                      @Kwesidei not the whole crew \n",
              "\n",
              "[5 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sIMz0fbF4tvI"
      },
      "source": [
        "## Checking for Repitative values."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JnxPMULJz9oD",
        "outputId": "fdf7f28a-47de-448e-fcf1-ccf1fe48b781",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "_data.shape"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1599999, 6)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KQJYuCwR5mRm",
        "outputId": "d33a5de1-03f7-413d-de61-a08f0a86296e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "_data.ids.nunique()"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1598314"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AbTi3SVY4_2z"
      },
      "source": [
        "### Here we can see that nearly 1685 ids are either missing or are duplicated or repititive."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0WPoI3NuxrMM",
        "outputId": "a08ac69a-a765-487b-9c2e-20edce8b9a1b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "_data.ids.isnull().sum()"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9ftbmi2E1EF1",
        "outputId": "62994a88-d388-404d-f9ae-cf24127df1fa",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "_data.ids.value_counts()"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1753678185    2\n",
              "1678693526    2\n",
              "1835870889    2\n",
              "2190104868    2\n",
              "1984377787    2\n",
              "             ..\n",
              "2056931811    1\n",
              "2212131300    1\n",
              "2188589814    1\n",
              "1983537638    1\n",
              "1964587131    1\n",
              "Name: ids, Length: 1598314, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1SFDPucG6aTp",
        "outputId": "705f22d6-8d36-4f50-96c2-28a6ee3bba18",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "_data.ids.value_counts().unique()"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([2, 1])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I1mu-Pxh6lGd",
        "outputId": "7a6230f0-3e3d-416e-df22-7390806b511c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 112
        }
      },
      "source": [
        "_data[_data['ids']==1753678185]"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>target</th>\n",
              "      <th>ids</th>\n",
              "      <th>date</th>\n",
              "      <th>flag</th>\n",
              "      <th>user</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>84765</th>\n",
              "      <td>0</td>\n",
              "      <td>1753678185</td>\n",
              "      <td>Sun May 10 02:51:55 PDT 2009</td>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>BoydyxO</td>\n",
              "      <td>sunnn finnalllyyy!!  aint slept :| need some s...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>918538</th>\n",
              "      <td>4</td>\n",
              "      <td>1753678185</td>\n",
              "      <td>Sun May 10 02:51:55 PDT 2009</td>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>BoydyxO</td>\n",
              "      <td>sunnn finnalllyyy!!  aint slept :| need some s...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        target  ...                                               text\n",
              "84765        0  ...  sunnn finnalllyyy!!  aint slept :| need some s...\n",
              "918538       4  ...  sunnn finnalllyyy!!  aint slept :| need some s...\n",
              "\n",
              "[2 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZVxT-uqV64ni",
        "outputId": "e105edc2-8269-43e0-aad2-34fc01285ea1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 112
        }
      },
      "source": [
        "_data[_data['ids']==1984377787]"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>target</th>\n",
              "      <th>ids</th>\n",
              "      <th>date</th>\n",
              "      <th>flag</th>\n",
              "      <th>user</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>254973</th>\n",
              "      <td>0</td>\n",
              "      <td>1984377787</td>\n",
              "      <td>Sun May 31 14:59:31 PDT 2009</td>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>15AMR</td>\n",
              "      <td>I'm broke cuz she got all she wanted! Going ho...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1194018</th>\n",
              "      <td>4</td>\n",
              "      <td>1984377787</td>\n",
              "      <td>Sun May 31 14:59:31 PDT 2009</td>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>15AMR</td>\n",
              "      <td>I'm broke cuz she got all she wanted! Going ho...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "         target  ...                                               text\n",
              "254973        0  ...  I'm broke cuz she got all she wanted! Going ho...\n",
              "1194018       4  ...  I'm broke cuz she got all she wanted! Going ho...\n",
              "\n",
              "[2 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_eNLzcRP70pK",
        "outputId": "5bfbda59-ec54-4fe1-a78f-77a4670bfb6d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 112
        }
      },
      "source": [
        "_data[_data['ids']==1835870889]"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>target</th>\n",
              "      <th>ids</th>\n",
              "      <th>date</th>\n",
              "      <th>flag</th>\n",
              "      <th>user</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>132902</th>\n",
              "      <td>0</td>\n",
              "      <td>1835870889</td>\n",
              "      <td>Mon May 18 06:55:21 PDT 2009</td>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>lillekerohus</td>\n",
              "      <td>I'm so nervous about my guitar exam tomorrow. ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>996346</th>\n",
              "      <td>4</td>\n",
              "      <td>1835870889</td>\n",
              "      <td>Mon May 18 06:55:21 PDT 2009</td>\n",
              "      <td>NO_QUERY</td>\n",
              "      <td>lillekerohus</td>\n",
              "      <td>I'm so nervous about my guitar exam tomorrow. ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        target  ...                                               text\n",
              "132902       0  ...  I'm so nervous about my guitar exam tomorrow. ...\n",
              "996346       4  ...  I'm so nervous about my guitar exam tomorrow. ...\n",
              "\n",
              "[2 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p4v6oNRk7Cs2"
      },
      "source": [
        "### from the above cells we can observe that there are no missing values present but some id values are not unique and are repeated.The whole tweet is repeated and no tweet is repeated more than twice and the repeated tweets have both positive and negative sentiment labelled on it."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9aQkQiFSkIqW"
      },
      "source": [
        "_data.set_index('ids',inplace=True)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YMzc1milCjoj",
        "outputId": "1cd07604-7d19-42cc-a0a0-676652b53d08",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "t=_data.index.value_counts()==2\n",
        "temp=t.loc[t[t.index]==True]\n",
        "dup_indexes=temp.index\n",
        "#dup_data=covid_data.loc[covid_data.ids\n",
        "print(dup_indexes)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Int64Index([1753678185, 1678693526, 1835870889, 2190104868, 1984377787,\n",
            "            1972032864, 2062313315, 2057992213, 1978945483, 1827871029,\n",
            "            ...\n",
            "            2191418783, 2013668899, 2190980212, 2051228740, 2059765907,\n",
            "            2044816045, 1971122723, 1957740028, 2015887552, 1835205913],\n",
            "           dtype='int64', length=1685)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EpFyoPQ-icYK"
      },
      "source": [
        "_data.drop(list(dup_indexes),inplace=True)\n"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9lgg7uphlKOS",
        "outputId": "b33bb372-dcd7-431c-b637-d67522b17ccc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "_data.shape"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1596629, 5)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uqHI-N8FZvHI"
      },
      "source": [
        "## Selecting only Relevant Columns."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AXshQDCtPio7",
        "outputId": "515b3141-9477-4d17-e40f-45378fd944fb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 237
        }
      },
      "source": [
        "_data_mod=_data[['target','date','text']]\n",
        "_data_mod.head()"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>target</th>\n",
              "      <th>date</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>ids</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1467810672</th>\n",
              "      <td>0</td>\n",
              "      <td>Mon Apr 06 22:19:49 PDT 2009</td>\n",
              "      <td>is upset that he can't update his Facebook by ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1467810917</th>\n",
              "      <td>0</td>\n",
              "      <td>Mon Apr 06 22:19:53 PDT 2009</td>\n",
              "      <td>@Kenichan I dived many times for the ball. Man...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1467811184</th>\n",
              "      <td>0</td>\n",
              "      <td>Mon Apr 06 22:19:57 PDT 2009</td>\n",
              "      <td>my whole body feels itchy and like its on fire</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1467811193</th>\n",
              "      <td>0</td>\n",
              "      <td>Mon Apr 06 22:19:57 PDT 2009</td>\n",
              "      <td>@nationwideclass no, it's not behaving at all....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1467811372</th>\n",
              "      <td>0</td>\n",
              "      <td>Mon Apr 06 22:20:00 PDT 2009</td>\n",
              "      <td>@Kwesidei not the whole crew</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            target  ...                                               text\n",
              "ids                 ...                                                   \n",
              "1467810672       0  ...  is upset that he can't update his Facebook by ...\n",
              "1467810917       0  ...  @Kenichan I dived many times for the ball. Man...\n",
              "1467811184       0  ...    my whole body feels itchy and like its on fire \n",
              "1467811193       0  ...  @nationwideclass no, it's not behaving at all....\n",
              "1467811372       0  ...                      @Kwesidei not the whole crew \n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sEf0oZFhZ4d_"
      },
      "source": [
        "## Checking for null values in the remaining columns.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LYORQvxWPzmy",
        "outputId": "daee5c88-cd74-4e75-ea34-a626c368dbc7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "_data_mod.isnull().sum()"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "target    0\n",
              "date      0\n",
              "text      0\n",
              "dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "toSGmN39aRhp"
      },
      "source": [
        "## Converting the dates into a Standard Format."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rNhDMlt-QGSn",
        "outputId": "18619a26-0bfe-43fe-dc5b-350528cc2931",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "_data_mod['date']=_data_mod['date'].apply(lambda x: pd.to_datetime(x).strftime('%m/%d/%Y'))"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/dateutil/parser/_parser.py:1218: UnknownTimezoneWarning: tzname PDT identified but not understood.  Pass `tzinfos` argument in order to correctly return a timezone-aware datetime.  In a future version, this will raise an exception.\n",
            "  category=UnknownTimezoneWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  \"\"\"Entry point for launching an IPython kernel.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LjpSeTTwQcS8",
        "outputId": "10460c8a-d82d-4744-c8cd-04be60bba9bd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 237
        }
      },
      "source": [
        "_data_mod.head()"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>target</th>\n",
              "      <th>date</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>ids</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1467810672</th>\n",
              "      <td>0</td>\n",
              "      <td>04/06/2009</td>\n",
              "      <td>is upset that he can't update his Facebook by ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1467810917</th>\n",
              "      <td>0</td>\n",
              "      <td>04/06/2009</td>\n",
              "      <td>@Kenichan I dived many times for the ball. Man...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1467811184</th>\n",
              "      <td>0</td>\n",
              "      <td>04/06/2009</td>\n",
              "      <td>my whole body feels itchy and like its on fire</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1467811193</th>\n",
              "      <td>0</td>\n",
              "      <td>04/06/2009</td>\n",
              "      <td>@nationwideclass no, it's not behaving at all....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1467811372</th>\n",
              "      <td>0</td>\n",
              "      <td>04/06/2009</td>\n",
              "      <td>@Kwesidei not the whole crew</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            target  ...                                               text\n",
              "ids                 ...                                                   \n",
              "1467810672       0  ...  is upset that he can't update his Facebook by ...\n",
              "1467810917       0  ...  @Kenichan I dived many times for the ball. Man...\n",
              "1467811184       0  ...    my whole body feels itchy and like its on fire \n",
              "1467811193       0  ...  @nationwideclass no, it's not behaving at all....\n",
              "1467811372       0  ...                      @Kwesidei not the whole crew \n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VnpzCViWoGB7"
      },
      "source": [
        "### Shuffling the Dataset."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9s5NF34snuQK",
        "outputId": "20f901d9-6f0c-4345-ec9c-f0085b28a645",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "_data_mod=_data_mod.sample(frac=1)\n",
        "_data_mod.target.values[:50]"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 4, 0, 0, 4, 0, 0, 4, 4, 0, 0, 0, 4, 0, 0, 4, 4, 0, 4, 4, 0, 4,\n",
              "       0, 4, 4, 4, 0, 4, 4, 0, 4, 4, 4, 0, 0, 4, 4, 0, 0, 4, 0, 4, 4, 0,\n",
              "       4, 4, 0, 4, 4, 4])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Kd83f6zbaZzf"
      },
      "source": [
        "## Serializing the DataFrame."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OI8ivre6oUUZ",
        "outputId": "24ed1679-62bd-4ac0-efd6-e930b0d27d0b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "source": [
        "os.getcwd()"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'/content/gdrive/My Drive/Colab Notebooks'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LMHz-ck_SzWP"
      },
      "source": [
        "_data_mod.to_pickle(\"_data_mod_pickled\")"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pkXnboMVYU62"
      },
      "source": [
        "data=pd.read_pickle(\"_data_mod_pickled\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LfXBSKPSivfA"
      },
      "source": [
        "data.target=data.target.replace({0:0,4:1})"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TpnshM2nfR-x",
        "outputId": "bf33a000-18da-400e-b236-bdb67000223f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        }
      },
      "source": [
        "data.target.value_counts()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1    798315\n",
              "0    798314\n",
              "Name: target, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F1-3SHnchqZv",
        "outputId": "1d5ddb33-5e04-45b3-92eb-7ad1e0e5acf1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "possible_labels=data.target.unique()\n",
        "possible_labels"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 1])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mrGl9xipkfjf"
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "x_train,x_val,y_train,y_val=train_test_split(data.index.values,data.target.values,test_size=0.20,random_state=11,stratify=data.target.values)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JW_Z9Mw1m5d3"
      },
      "source": [
        "data['data_type']=['not_set']*data.shape[0]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iB74mXG3plG1",
        "outputId": "8521a1c5-d4d8-4ea0-ee20-840f125a3de9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 109
        }
      },
      "source": [
        "data.data_type.loc[x_train]='train'\n",
        "data.data_type.loc[x_val]='val'"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  iloc._setitem_with_indexer(indexer, value)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MlRnxt1VnY-d",
        "outputId": "64d089bd-4ffe-4042-9779-879d42a012fd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        }
      },
      "source": [
        "data.groupby(['target','data_type']).count()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>target</th>\n",
              "      <th>data_type</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">0</th>\n",
              "      <th>train</th>\n",
              "      <td>638651</td>\n",
              "      <td>638651</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>val</th>\n",
              "      <td>159663</td>\n",
              "      <td>159663</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
              "      <th>train</th>\n",
              "      <td>638652</td>\n",
              "      <td>638652</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>val</th>\n",
              "      <td>159663</td>\n",
              "      <td>159663</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                    date    text\n",
              "target data_type                \n",
              "0      train      638651  638651\n",
              "       val        159663  159663\n",
              "1      train      638652  638652\n",
              "       val        159663  159663"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2vj-5SZ3EwxF"
      },
      "source": [
        "path='/content/gdrive/My Drive/Colab Notebooks/data_chunks'\n",
        "os.chdir(path)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T8c8DfVqCk-F"
      },
      "source": [
        "#os.mkdir(\"data_chunks\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DBY51yiPFs8B",
        "outputId": "c00ae397-bdfb-421f-89ca-d402ef0bf90c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "source": [
        "os.getcwd()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'/content/gdrive/My Drive/Colab Notebooks/data_chunks'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6-3H3mnhvTPZ"
      },
      "source": [
        "train_data=data.loc[x_train]\n",
        "train_data.drop(['date'],axis=1,inplace=True)\n",
        "val_data=data.loc[x_val]\n",
        "val_data.drop(['date'],axis=1,inplace=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LpuwSZa9vdcm",
        "outputId": "a915526b-1bea-4c0e-c4d6-fb71dcf231a2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "len(train_data.target.values==y_train)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1277303"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6BwiiMLvjpaD",
        "outputId": "13302f6b-5485-49b4-a2bd-e386efc7d21c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "len(val_data.target.values==y_val)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "319326"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yh3PzXp4wEP5",
        "outputId": "d9443b00-1031-45ec-ce3f-e851806f1f8d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "x_train.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1277303,)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r3C90AVOv8TV",
        "outputId": "1ebcb26e-c359-4e92-d5d0-fe95e50130fb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "y_train.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1277303,)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m4QJKYYsDvv3",
        "outputId": "af0dfd62-c21c-46a1-fd96-243ae98b6017",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 479
        }
      },
      "source": [
        "i=0\n",
        "chunk_size=50000\n",
        "count=1\n",
        "while count<26:\n",
        "  if count<25:\n",
        "    chunk=train_data.iloc[i:chunk_size+i,:]\n",
        "    i=i+chunk_size\n",
        "  else:\n",
        "    chunk=train_data.iloc[i:,:]\n",
        "  chunk.to_pickle(\"train_chunk_{}\".format(count))\n",
        "  print(\"finished working on train_chunk_{}\".format(count))\n",
        "  count+=1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "finished working on train_chunk_1\n",
            "finished working on train_chunk_2\n",
            "finished working on train_chunk_3\n",
            "finished working on train_chunk_4\n",
            "finished working on train_chunk_5\n",
            "finished working on train_chunk_6\n",
            "finished working on train_chunk_7\n",
            "finished working on train_chunk_8\n",
            "finished working on train_chunk_9\n",
            "finished working on train_chunk_10\n",
            "finished working on train_chunk_11\n",
            "finished working on train_chunk_12\n",
            "finished working on train_chunk_13\n",
            "finished working on train_chunk_14\n",
            "finished working on train_chunk_15\n",
            "finished working on train_chunk_16\n",
            "finished working on train_chunk_17\n",
            "finished working on train_chunk_18\n",
            "finished working on train_chunk_19\n",
            "finished working on train_chunk_20\n",
            "finished working on train_chunk_21\n",
            "finished working on train_chunk_22\n",
            "finished working on train_chunk_23\n",
            "finished working on train_chunk_24\n",
            "finished working on train_chunk_25\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9F47xSbhS6zp",
        "outputId": "622ae70b-ca92-4f06-aeb5-cb626e9e0bbd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 146
        }
      },
      "source": [
        "i=0\n",
        "chunk_size=50000\n",
        "count=1\n",
        "while count<8:\n",
        "  if count<7:\n",
        "    chunk=val_data.iloc[i:chunk_size+i,:]\n",
        "    i=i+chunk_size\n",
        "  else:\n",
        "    chunk=val_data.iloc[i:,:]\n",
        "  chunk.to_pickle(\"val_chunk_{}\".format(count))\n",
        "  print(\"finished working on val_chunk_{}\".format(count))\n",
        "  count+=1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "finished working on val_chunk_1\n",
            "finished working on val_chunk_2\n",
            "finished working on val_chunk_3\n",
            "finished working on val_chunk_4\n",
            "finished working on val_chunk_5\n",
            "finished working on val_chunk_6\n",
            "finished working on val_chunk_7\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}