Text classification with XLNET.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d8fe0306-d7bd-4b60-b269-51e26ed1cf99",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lnewb\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from cleantext import clean\n",
    "import re\n",
    "from transformers import XLNetTokenizer, XLNetForSequenceClassification, TrainingArguments, Trainer, pipeline\n",
    "import torch\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import datasets \n",
    "import evaluate\n",
    "import random"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06919424-e442-4bff-b4cd-32c1a1cf48a5",
   "metadata": {},
   "source": [
    "## Preprocess our data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c0b354e5-4e04-4731-b63e-4174746ac16b",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_train = pd.read_csv('./emotions_data/emotion-labels-train.csv')\n",
    "data_test = pd.read_csv('./emotions_data/emotion-labels-test.csv')\n",
    "data_val = pd.read_csv('./emotions_data/emotion-labels-val.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "94b72ed5-f2ae-4ba8-94be-2c769a5fb424",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Just got back from seeing @GaryDelaney in Burs...</td>\n",
       "      <td>joy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Oh dear an evening of absolute hilarity I don'...</td>\n",
       "      <td>joy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Been waiting all week for this game ❤️❤️❤️ #ch...</td>\n",
       "      <td>joy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>@gardiner_love : Thank you so much, Gloria! Yo...</td>\n",
       "      <td>joy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I feel so blessed to work with the family that...</td>\n",
       "      <td>joy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text label\n",
       "0  Just got back from seeing @GaryDelaney in Burs...   joy\n",
       "1  Oh dear an evening of absolute hilarity I don'...   joy\n",
       "2  Been waiting all week for this game ❤️❤️❤️ #ch...   joy\n",
       "3  @gardiner_love : Thank you so much, Gloria! Yo...   joy\n",
       "4  I feel so blessed to work with the family that...   joy"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2d9dcbd9-00f2-4436-9537-13068f4b03bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.concat([data_train, data_test, data_val], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "96b6c800-9b42-4fea-8ab2-0b0579a8a14c",
   "metadata": {},
   "outputs": [],
   "source": [
    "data['text_clean'] = data['text'].apply(lambda x: clean(x, no_emoji=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4e524c66-0beb-4567-af7a-dc1f1651c60d",
   "metadata": {},
   "outputs": [],
   "source": [
    "data['text_clean'] = data['text_clean'].apply(lambda x: re.sub('@[^\\s]+', '', x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9e8eac14-1091-4cb2-8e1b-bf1ea167eb50",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>text_clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Just got back from seeing @GaryDelaney in Burs...</td>\n",
       "      <td>joy</td>\n",
       "      <td>just got back from seeing  in burslem. amazing...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Oh dear an evening of absolute hilarity I don'...</td>\n",
       "      <td>joy</td>\n",
       "      <td>oh dear an evening of absolute hilarity i don'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Been waiting all week for this game ❤️❤️❤️ #ch...</td>\n",
       "      <td>joy</td>\n",
       "      <td>been waiting all week for this game #cheer #fr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>@gardiner_love : Thank you so much, Gloria! Yo...</td>\n",
       "      <td>joy</td>\n",
       "      <td>: thank you so much, gloria! you're so sweet,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I feel so blessed to work with the family that...</td>\n",
       "      <td>joy</td>\n",
       "      <td>i feel so blessed to work with the family that...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Today I reached 1000 subscribers on YT!! , #go...</td>\n",
       "      <td>joy</td>\n",
       "      <td>today i reached 1000 subscribers on yt!! , #go...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>@Singaholic121 Good morning, love! Happy first...</td>\n",
       "      <td>joy</td>\n",
       "      <td>good morning, love! happy first day of fall. ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>#BridgetJonesBaby is the best thing I've seen ...</td>\n",
       "      <td>joy</td>\n",
       "      <td>#bridgetjonesbaby is the best thing i've seen ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Just got back from seeing @GaryDelaney in Burs...</td>\n",
       "      <td>joy</td>\n",
       "      <td>just got back from seeing  in burslem. amazing...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>@IndyMN I thought the holidays could not get a...</td>\n",
       "      <td>joy</td>\n",
       "      <td>i thought the holidays could not get any more...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>I'm just still . So happy .\\nA blast</td>\n",
       "      <td>joy</td>\n",
       "      <td>i'm just still . so happy .\\na blast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>It's meant to be!! #happy #happy</td>\n",
       "      <td>joy</td>\n",
       "      <td>it's meant to be!! #happy #happy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>💥⚖️Yeah‼️ PAUL‼️⚖️💥  #glorious #BB18</td>\n",
       "      <td>joy</td>\n",
       "      <td>yeah!! paul!! #glorious #bb18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>My morning started off amazing!! Hopefully the...</td>\n",
       "      <td>joy</td>\n",
       "      <td>my morning started off amazing!! hopefully the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>😱 @cailamarsai you've had me 😂 😂 the whole tim...</td>\n",
       "      <td>joy</td>\n",
       "      <td>you've had me the whole time watching  after ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>@iamTinaDatta love you so much #smile 😊😊</td>\n",
       "      <td>joy</td>\n",
       "      <td>love you so much #smile</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>@WyoWiseGuy @LivingVertical however, REI did o...</td>\n",
       "      <td>joy</td>\n",
       "      <td>however, rei did offer me the job today as w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>2 days until #GoPackGo and 23 days until #GoGi...</td>\n",
       "      <td>joy</td>\n",
       "      <td>2 days until #gopackgo and 23 days until #gogi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>@TheMandyMoore You are beyond wonderful.  Your...</td>\n",
       "      <td>joy</td>\n",
       "      <td>you are beyond wonderful. your singing prowes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>@luckiiCHARM_ Luckii, I'm changing in so many ...</td>\n",
       "      <td>joy</td>\n",
       "      <td>luckii, i'm changing in so many ways bc of hi...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 text label  \\\n",
       "0   Just got back from seeing @GaryDelaney in Burs...   joy   \n",
       "1   Oh dear an evening of absolute hilarity I don'...   joy   \n",
       "2   Been waiting all week for this game ❤️❤️❤️ #ch...   joy   \n",
       "3   @gardiner_love : Thank you so much, Gloria! Yo...   joy   \n",
       "4   I feel so blessed to work with the family that...   joy   \n",
       "5   Today I reached 1000 subscribers on YT!! , #go...   joy   \n",
       "6   @Singaholic121 Good morning, love! Happy first...   joy   \n",
       "7   #BridgetJonesBaby is the best thing I've seen ...   joy   \n",
       "8   Just got back from seeing @GaryDelaney in Burs...   joy   \n",
       "9   @IndyMN I thought the holidays could not get a...   joy   \n",
       "10               I'm just still . So happy .\\nA blast   joy   \n",
       "11                   It's meant to be!! #happy #happy   joy   \n",
       "12               💥⚖️Yeah‼️ PAUL‼️⚖️💥  #glorious #BB18   joy   \n",
       "13  My morning started off amazing!! Hopefully the...   joy   \n",
       "14  😱 @cailamarsai you've had me 😂 😂 the whole tim...   joy   \n",
       "15           @iamTinaDatta love you so much #smile 😊😊   joy   \n",
       "16  @WyoWiseGuy @LivingVertical however, REI did o...   joy   \n",
       "17  2 days until #GoPackGo and 23 days until #GoGi...   joy   \n",
       "18  @TheMandyMoore You are beyond wonderful.  Your...   joy   \n",
       "19  @luckiiCHARM_ Luckii, I'm changing in so many ...   joy   \n",
       "\n",
       "                                           text_clean  \n",
       "0   just got back from seeing  in burslem. amazing...  \n",
       "1   oh dear an evening of absolute hilarity i don'...  \n",
       "2   been waiting all week for this game #cheer #fr...  \n",
       "3    : thank you so much, gloria! you're so sweet,...  \n",
       "4   i feel so blessed to work with the family that...  \n",
       "5   today i reached 1000 subscribers on yt!! , #go...  \n",
       "6    good morning, love! happy first day of fall. ...  \n",
       "7   #bridgetjonesbaby is the best thing i've seen ...  \n",
       "8   just got back from seeing  in burslem. amazing...  \n",
       "9    i thought the holidays could not get any more...  \n",
       "10               i'm just still . so happy .\\na blast  \n",
       "11                   it's meant to be!! #happy #happy  \n",
       "12                      yeah!! paul!! #glorious #bb18  \n",
       "13  my morning started off amazing!! hopefully the...  \n",
       "14   you've had me the whole time watching  after ...  \n",
       "15                            love you so much #smile  \n",
       "16    however, rei did offer me the job today as w...  \n",
       "17  2 days until #gopackgo and 23 days until #gogi...  \n",
       "18   you are beyond wonderful. your singing prowes...  \n",
       "19   luckii, i'm changing in so many ways bc of hi...  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "29576bb4-ac90-43be-be85-a9c12a22da4f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: xlabel='label'>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAHbCAYAAADVmqCzAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAprklEQVR4nO3de1SVdb7H8Q8XwevehArIhPf7Ba+FnLyljKhkms7yOHJMTa1coCmljmsVXuqkmZlRJuN0zGy0LFOn1FBGx0uKFzQ073kFRwFLAcEEFc4fLfe0UysNePYP3q+19lrxPD/Y391Oevs8z97braioqEgAAAAGcbd6AAAAgHtFwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwjqfVA5SUwsJCnT9/XtWqVZObm5vV4wAAgN+gqKhIV65cUWBgoNzd736cpcwGzPnz5xUUFGT1GAAA4D6kpaXpwQcfvOv+Mhsw1apVk/TjvwCbzWbxNAAA4LfIyclRUFCQ4//jd1NmA+bWaSObzUbAAABgmF+7/IOLeAEAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGMfT6gHKqrp/WWv1CJY4MyvC6hEAAOUAR2AAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAce4pYGbOnKmHHnpI1apVk5+fn/r3769jx445rbl27ZqioqJUvXp1Va1aVQMHDlRGRobTmtTUVEVERKhy5cry8/PTxIkTdePGDac1mzdvVrt27eTt7a2GDRtq8eLF9/cIAQBAmXNPAbNlyxZFRUVp586dSkxM1PXr19WzZ0/l5eU51kyYMEFffPGFPv30U23ZskXnz5/XgAEDHPtv3rypiIgIFRQUaMeOHfrggw+0ePFixcbGOtacPn1aERERevTRR5WSkqLx48dr1KhRWr9+fTE8ZAAAYDq3oqKiovv95osXL8rPz09btmxRly5dlJ2drZo1a2rZsmX605/+JEk6evSomjVrpqSkJHXs2FFffvmlHnvsMZ0/f17+/v6SpPj4eE2ePFkXL16Ul5eXJk+erLVr1+rgwYOO+xo8eLCysrKUkJDwm2bLycmR3W5Xdna2bDbb/T7E+1b3L2tL/T5dwZlZEVaPAAAw2G/9//fvugYmOztbkuTr6ytJ2rt3r65fv66wsDDHmqZNm6p27dpKSkqSJCUlJalVq1aOeJGk8PBw5eTk6NChQ441P/0Zt9bc+hkAAKB887zfbywsLNT48eP1yCOPqGXLlpKk9PR0eXl5ycfHx2mtv7+/0tPTHWt+Gi+39t/a90trcnJy9MMPP6hSpUq3zZOfn6/8/HzH1zk5Off70AAAgIu77yMwUVFROnjwoD7++OPinOe+zZw5U3a73XELCgqyeiQAAFBC7itgoqOjtWbNGv3rX//Sgw8+6NgeEBCggoICZWVlOa3PyMhQQECAY83PX5V06+tfW2Oz2e549EWSpkyZouzsbMctLS3tfh4aAAAwwD0FTFFRkaKjo7Vq1Spt2rRJ9erVc9rfvn17VahQQRs3bnRsO3bsmFJTUxUaGipJCg0N1TfffKPMzEzHmsTERNlsNjVv3tyx5qc/49aaWz/jTry9vWWz2ZxuAACgbLqna2CioqK0bNky/eMf/1C1atUc16zY7XZVqlRJdrtdI0eOVExMjHx9fWWz2TR27FiFhoaqY8eOkqSePXuqefPmGjp0qGbPnq309HS9+OKLioqKkre3tyTp2Wef1TvvvKNJkybpqaee0qZNm/TJJ59o7dry+coeAADg7J6OwCxYsEDZ2dnq1q2batWq5bgtX77csebNN9/UY489poEDB6pLly4KCAjQypUrHfs9PDy0Zs0aeXh4KDQ0VP/zP/+jJ598UjNmzHCsqVevntauXavExES1bt1ab7zxht577z2Fh4cXw0MGAACm+13vA+PKeB8Ya/A+MACA36NU3gcGAADACgQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4nlYPAJQFdf+y1uoRLHFmVoTVIwAopzgCAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4f5ggA94gP7wSsxxEYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcfgwRwAAfgEf3uma7vkIzNatW9W3b18FBgbKzc1Nq1evdto/fPhwubm5Od169erltObSpUuKjIyUzWaTj4+PRo4cqdzcXKc1Bw4cUOfOnVWxYkUFBQVp9uzZ9/7oAABAmXTPAZOXl6fWrVtr/vz5d13Tq1cvXbhwwXH76KOPnPZHRkbq0KFDSkxM1Jo1a7R161Y9/fTTjv05OTnq2bOn6tSpo7179+r111/XtGnTtHDhwnsdFwAAlEH3fAqpd+/e6t279y+u8fb2VkBAwB33HTlyRAkJCdqzZ486dOggSXr77bfVp08fzZkzR4GBgVq6dKkKCgq0aNEieXl5qUWLFkpJSdHcuXOdQgcAAJRPJXIR7+bNm+Xn56cmTZpozJgx+v777x37kpKS5OPj44gXSQoLC5O7u7t27drlWNOlSxd5eXk51oSHh+vYsWO6fPnyHe8zPz9fOTk5TjcAAFA2FXvA9OrVS0uWLNHGjRv12muvacuWLerdu7du3rwpSUpPT5efn5/T93h6esrX11fp6emONf7+/k5rbn19a83PzZw5U3a73XELCgoq7ocGAABcRLG/Cmnw4MGOf27VqpWCg4PVoEEDbd68WT169Cjuu3OYMmWKYmJiHF/n5OQQMQAAlFEl/j4w9evXV40aNXTixAlJUkBAgDIzM53W3LhxQ5cuXXJcNxMQEKCMjAynNbe+vtu1Nd7e3rLZbE43AABQNpV4wJw7d07ff/+9atWqJUkKDQ1VVlaW9u7d61izadMmFRYWKiQkxLFm69atun79umNNYmKimjRpogceeKCkRwYAAC7ungMmNzdXKSkpSklJkSSdPn1aKSkpSk1NVW5uriZOnKidO3fqzJkz2rhxo/r166eGDRsqPDxcktSsWTP16tVLo0eP1u7du7V9+3ZFR0dr8ODBCgwMlCQNGTJEXl5eGjlypA4dOqTly5frrbfecjpFBAAAyq97Dpjk5GS1bdtWbdu2lSTFxMSobdu2io2NlYeHhw4cOKDHH39cjRs31siRI9W+fXtt27ZN3t7ejp+xdOlSNW3aVD169FCfPn3UqVMnp/d4sdvt2rBhg06fPq327dvr+eefV2xsLC+hBgAAku7jIt5u3bqpqKjorvvXr1//qz/D19dXy5Yt+8U1wcHB2rZt272OBwAAygE+zBEAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAce45YLZu3aq+ffsqMDBQbm5uWr16tdP+oqIixcbGqlatWqpUqZLCwsL07bffOq25dOmSIiMjZbPZ5OPjo5EjRyo3N9dpzYEDB9S5c2dVrFhRQUFBmj179r0/OgAAUCbdc8Dk5eWpdevWmj9//h33z549W3FxcYqPj9euXbtUpUoVhYeH69q1a441kZGROnTokBITE7VmzRpt3bpVTz/9tGN/Tk6OevbsqTp16mjv3r16/fXXNW3aNC1cuPA+HiIAAChrPO/1G3r37q3evXvfcV9RUZHmzZunF198Uf369ZMkLVmyRP7+/lq9erUGDx6sI0eOKCEhQXv27FGHDh0kSW+//bb69OmjOXPmKDAwUEuXLlVBQYEWLVokLy8vtWjRQikpKZo7d65T6AAAgPKpWK+BOX36tNLT0xUWFubYZrfbFRISoqSkJElSUlKSfHx8HPEiSWFhYXJ3d9euXbsca7p06SIvLy/HmvDwcB07dkyXL1++433n5+crJyfH6QYAAMqmYg2Y9PR0SZK/v7/Tdn9/f8e+9PR0+fn5Oe339PSUr6+v05o7/Yyf3sfPzZw5U3a73XELCgr6/Q8IAAC4pDLzKqQpU6YoOzvbcUtLS7N6JAAAUEKKNWACAgIkSRkZGU7bMzIyHPsCAgKUmZnptP/GjRu6dOmS05o7/Yyf3sfPeXt7y2azOd0AAEDZVKwBU69ePQUEBGjjxo2ObTk5Odq1a5dCQ0MlSaGhocrKytLevXsdazZt2qTCwkKFhIQ41mzdulXXr193rElMTFSTJk30wAMPFOfIAADAQPccMLm5uUpJSVFKSoqkHy/cTUlJUWpqqtzc3DR+/Hi98sor+vzzz/XNN9/oySefVGBgoPr37y9JatasmXr16qXRo0dr9+7d2r59u6KjozV48GAFBgZKkoYMGSIvLy+NHDlShw4d0vLly/XWW28pJiam2B44AAAw1z2/jDo5OVmPPvqo4+tbUTFs2DAtXrxYkyZNUl5enp5++mllZWWpU6dOSkhIUMWKFR3fs3TpUkVHR6tHjx5yd3fXwIEDFRcX59hvt9u1YcMGRUVFqX379qpRo4ZiY2N5CTUAAJB0HwHTrVs3FRUV3XW/m5ubZsyYoRkzZtx1ja+vr5YtW/aL9xMcHKxt27bd63gAAKAcKDOvQgIAAOUHAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xR7wEybNk1ubm5Ot6ZNmzr2X7t2TVFRUapevbqqVq2qgQMHKiMjw+lnpKamKiIiQpUrV5afn58mTpyoGzduFPeoAADAUJ4l8UNbtGihf/7zn/+5E8//3M2ECRO0du1affrpp7Lb7YqOjtaAAQO0fft2SdLNmzcVERGhgIAA7dixQxcuXNCTTz6pChUq6NVXXy2JcQEAgGFKJGA8PT0VEBBw2/bs7Gz93//9n5YtW6bu3btLkt5//301a9ZMO3fuVMeOHbVhwwYdPnxY//znP+Xv7682bdro5Zdf1uTJkzVt2jR5eXmVxMgAAMAgJXINzLfffqvAwEDVr19fkZGRSk1NlSTt3btX169fV1hYmGNt06ZNVbt2bSUlJUmSkpKS1KpVK/n7+zvWhIeHKycnR4cOHbrrfebn5ysnJ8fpBgAAyqZiD5iQkBAtXrxYCQkJWrBggU6fPq3OnTvrypUrSk9Pl5eXl3x8fJy+x9/fX+np6ZKk9PR0p3i5tf/WvruZOXOm7Ha74xYUFFS8DwwAALiMYj+F1Lt3b8c/BwcHKyQkRHXq1NEnn3yiSpUqFffdOUyZMkUxMTGOr3NycogYAADKqBJ/GbWPj48aN26sEydOKCAgQAUFBcrKynJak5GR4bhmJiAg4LZXJd36+k7X1dzi7e0tm83mdAMAAGVTiQdMbm6uTp48qVq1aql9+/aqUKGCNm7c6Nh/7NgxpaamKjQ0VJIUGhqqb775RpmZmY41iYmJstlsat68eUmPCwAADFDsp5BeeOEF9e3bV3Xq1NH58+c1depUeXh46M9//rPsdrtGjhypmJgY+fr6ymazaezYsQoNDVXHjh0lST179lTz5s01dOhQzZ49W+np6XrxxRcVFRUlb2/v4h4XAAAYqNgD5ty5c/rzn/+s77//XjVr1lSnTp20c+dO1axZU5L05ptvyt3dXQMHDlR+fr7Cw8P17rvvOr7fw8NDa9as0ZgxYxQaGqoqVapo2LBhmjFjRnGPCgAADFXsAfPxxx//4v6KFStq/vz5mj9//l3X1KlTR+vWrSvu0QAAQBnBZyEBAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMI5LB8z8+fNVt25dVaxYUSEhIdq9e7fVIwEAABfgsgGzfPlyxcTEaOrUqdq3b59at26t8PBwZWZmWj0aAACwmMsGzNy5czV69GiNGDFCzZs3V3x8vCpXrqxFixZZPRoAALCYSwZMQUGB9u7dq7CwMMc2d3d3hYWFKSkpycLJAACAK/C0eoA7+e6773Tz5k35+/s7bff399fRo0fv+D35+fnKz893fJ2dnS1JysnJKblBf0Fh/lVL7tdqVv37thrPd/nC812+8Hxbc79FRUW/uM4lA+Z+zJw5U9OnT79te1BQkAXTlF/2eVZPgNLE812+8HyXL1Y/31euXJHdbr/rfpcMmBo1asjDw0MZGRlO2zMyMhQQEHDH75kyZYpiYmIcXxcWFurSpUuqXr263NzcSnReV5KTk6OgoCClpaXJZrNZPQ5KGM93+cLzXb6U1+e7qKhIV65cUWBg4C+uc8mA8fLyUvv27bVx40b1799f0o9BsnHjRkVHR9/xe7y9veXt7e20zcfHp4QndV02m61c/Qdf3vF8ly883+VLeXy+f+nIyy0uGTCSFBMTo2HDhqlDhw56+OGHNW/ePOXl5WnEiBFWjwYAACzmsgHz3//937p48aJiY2OVnp6uNm3aKCEh4bYLewEAQPnjsgEjSdHR0Xc9ZYQ78/b21tSpU287nYayiee7fOH5Ll94vn+ZW9GvvU4JAADAxbjkG9kBAAD8EgIGAAAYh4ABAADGIWAAwEWdOnXK6hEAl0XAGOz69etq0KCBjhw5YvUoKCU3btzQjBkzdO7cOatHQSlo2LChHn30Uf3973/XtWvXrB4HpeCHH37Q1av/+eyls2fPat68edqwYYOFU7kmAsZgFSpU4JdaOePp6anXX39dN27csHoUlIJ9+/YpODhYMTExCggI0DPPPKPdu3dbPRZKUL9+/bRkyRJJUlZWlkJCQvTGG2+oX79+WrBggcXTuRYCxnBRUVF67bXX+B9aOdK9e3dt2bLF6jFQCtq0aaO33npL58+f16JFi3ThwgV16tRJLVu21Ny5c3Xx4kWrR0Qx27dvnzp37ixJWrFihfz9/XX27FktWbJEcXFxFk/nWngfGMM98cQT2rhxo6pWrapWrVqpSpUqTvtXrlxp0WQoKfHx8Zo+fboiIyPVvn37257zxx9/3KLJUNLy8/P17rvvasqUKSooKJCXl5cGDRqk1157TbVq1bJ6PBSDypUr6+jRo6pdu7YGDRqkFi1aaOrUqUpLS1OTJk2cTi+VdwSM4X7ts6Hef//9UpoEpcXd/e4HTt3c3HTz5s1SnAalITk5WYsWLdLHH3+sKlWqaNiwYRo5cqTOnTun6dOnKycnh1NLZURwcLBGjRqlJ554Qi1btlRCQoJCQ0O1d+9eRUREKD093eoRXQYBAwAuau7cuXr//fd17Ngx9enTR6NGjVKfPn2cIvbcuXOqW7cup5HLiBUrVmjIkCG6efOmevTo4bh4d+bMmdq6dau+/PJLiyd0HQQMYLBr166pYsWKVo+BEtKoUSM99dRTGj58+F1PERUUFOijjz7SsGHDSnk6lJT09HRduHBBrVu3dsTq7t27ZbPZ1LRpU4uncx0ETBmwYsUKffLJJ0pNTVVBQYHTvn379lk0FUrKzZs39eqrryo+Pl4ZGRk6fvy46tevr5deekl169bVyJEjrR4RQDHJycnRpk2b1KRJEzVr1szqcVwKr0IyXFxcnEaMGCF/f399/fXXevjhh1W9enWdOnVKvXv3tno8lID//d//1eLFizV79mx5eXk5trds2VLvvfeehZOhJGRlZemNN97QqFGjNGrUKL355pvKzs62eiyUkEGDBumdd96R9ON7wnTo0EGDBg1ScHCwPvvsM4uncy0EjOHeffddLVy4UG+//ba8vLw0adIkJSYmaty4cfySK6OWLFmihQsXKjIyUh4eHo7trVu31tGjRy2cDMUtOTlZDRo00JtvvqlLly7p0qVLmjt3rho0aMDR1TJq69atjpdRr1q1SkVFRcrKylJcXJxeeeUVi6dzLQSM4VJTU/Vf//VfkqRKlSrpypUrkqShQ4fqo48+snI0lJB///vfatiw4W3bCwsLdf36dQsmQkmZMGGCHn/8cZ05c0YrV67UypUrdfr0aT322GMaP3681eOhBGRnZ8vX11eSlJCQoIEDB6py5cqKiIjQt99+a/F0roWAMVxAQIAuXbokSapdu7Z27twpSTp9+rS4vKlsat68ubZt23bb9hUrVqht27YWTISSkpycrMmTJ8vT09OxzdPTU5MmTVJycrKFk6GkBAUFKSkpSXl5eUpISFDPnj0lSZcvX+aC/Z/x/PUlcGXdu3fX559/rrZt22rEiBGaMGGCVqxYoeTkZA0YMMDq8VACYmNjNWzYMP373/9WYWGhVq5cqWPHjmnJkiVas2aN1eOhGNlsNqWmpt72ypO0tDRVq1bNoqlQksaPH6/IyEhVrVpVtWvXVrdu3ST9eGqpVatW1g7nYngVkuEKCwtVWFjo+Bvaxx9/rB07dqhRo0Z65plnnC7yRNmxbds2zZgxQ/v371dubq7atWun2NhYx9/WUDaMGzdOq1at0pw5cxynirdv366JEydq4MCBmjdvnrUDokQkJycrLS1Nf/zjH1W1alVJ0tq1a+Xj46NHHnnE4ulcBwEDAC6qoKBAEydOVHx8vG7cuKGioiJ5eXlpzJgxmjVrlry9va0eESWkoKBAp0+fVoMGDZxOIeI/CJgyYNu2bfrrX/+qkydPasWKFfrDH/6gDz/8UPXq1VOnTp2sHg/A73T16lWdPHlSktSgQQNVrlzZ4olQUq5evaqxY8fqgw8+kCTH+zyNHTtWf/jDH/SXv/zF4gldB1lnuM8++0xDhw5VZGSkvv76a+Xn50v68Ur2V199VevWrbN4QhS3Bx54QG5ubrdtd3NzU8WKFdWwYUMNHz78Vz8nC65pwIABWrx4sWw2269ex1a1alW1aNFCzz77rOx2eylNiJI0ZcoU7d+/X5s3b1avXr0c28PCwjRt2jQC5id4FZLhXnnlFcXHx+tvf/ubKlSo4Nj+yCOP8D4RZVRsbKzc3d0VERGh6dOna/r06YqIiJC7u7uioqLUuHFjjRkzRn/729+sHhX3wW63OwLVbrf/4u3GjRuKj4/X0KFDLZ4axWX16tV655131KlTJ6e/qLRo0cJxFA4/4giM4Y4dO6YuXbrctt1utysrK6v0B0KJ++qrr/TKK6/o2Wefddr+17/+VRs2bNBnn32m4OBgxcXFafTo0RZNifv100+Q/y2fJn/48GE99NBDJTkSStHFixfl5+d32/a8vLw7HnktzzgCY7iAgACdOHHitu1fffWV6tevb8FEKGnr169XWFjYbdt79Oih9evXS5L69OmjU6dOlfZosECTJk20Y8cOq8dAMenQoYPWrl3r+PpWtLz33nsKDQ21aiyXxBEYw40ePVrPPfecFi1aJDc3N50/f15JSUl64YUX9NJLL1k9HkqAr6+vvvjiC02YMMFp+xdffOF4B8+8vDzeJ6Sc8PDwUOvWra0eA8Xk1VdfVe/evXX48GHduHFDb731lg4fPqwdO3Zoy5YtVo/nUggYAx04cEAtW7aUu7u7pkyZosLCQvXo0UNXr15Vly5d5O3trRdeeEFjx461elSUgJdeekljxozRv/71Lz388MOSpD179mjdunWKj4+XJCUmJqpr165WjgngPnTq1EkpKSmaNWuWWrVqpQ0bNqhdu3ZKSkrijex+hpdRG8jDw0MXLlyQn5+f6tevrz179qhatWo6ceKEcnNz1bx5c8ebH6Fs2r59u9555x0dO3ZM0o+nEcaOHet4szMAKOsIGANVr15d69atU0hIiNzd3ZWRkaGaNWtaPRYAoBgUFhbqxIkTyszMVGFhodO+O71oo7ziFJKBBg4cqK5du6pWrVpyc3NThw4d5OHhcce1XMhZNvELDiibdu7cqSFDhujs2bO3fSCvm5ubbt68adFkroeAMdDChQs1YMAAnThxQuPGjdPo0aO5YLMc4RccUHY9++yzjlci3fpLKu6MU0iGGzFihOLi4giYcqRNmzZq3Lixpk+ffsdfcLwjK2CuKlWqaP/+/WrYsKHVo7g8AgYwDL/ggLKre/fumjRpktPHCODOOIUEGCYkJEQnTpwgYIAyaOzYsXr++eeVnp6uVq1aOX1EjCQFBwdbNJnr4QgMYJhVq1bpxRdf1MSJE/kFB5Qx7u63v0G+m5ubioqKuMbtZwgYwDB3+gV3C7/gALOdPXv2F/fXqVOnlCZxfQQMYBh+wQEA18AAxrkVKIcPH1ZqaqoKCgoc+9zc3AgYwDCff/75b177+OOPl+AkZuEIDGCYU6dO6YknntA333zjODcu/edTazmFBJjl56eFf/rn+tbXt/Dn+z/ufjIdgEt67rnnVK9ePWVmZqpy5co6ePCgtm7dqg4dOmjz5s1WjwfgHhUWFjpuGzZsUJs2bfTll18qKytLWVlZWrdundq1a6eEhASrR3UpHIEBDFOjRg1t2rRJwcHBstvt2r17t5o0aaJNmzbp+eef19dff231iADuU8uWLRUfH69OnTo5bd+2bZuefvppHTlyxKLJXA9HYADD3Lx50/HOyzVq1ND58+cl/XhtzK1PpwZgppMnT8rHx+e27Xa7XWfOnCn1eVwZAQMYpmXLltq/f7+kH9/Ubvbs2dq+fbtmzJih+vXrWzwdgN/joYceUkxMjDIyMhzbMjIyNHHiRD388MMWTuZ6OIUEGGb9+vXKy8tzfKDnY489puPHj6t69epavny5unfvbvWIAO7TiRMn9MQTT+j48eMKCgqSJKWlpalRo0ZavXo178D9EwQMUAZcunRJDzzwAJ9cC5QBRUVFSkxM1NGjRyVJzZo1U1hYGH++f4aAAQAAxuGN7AAAcCF5eXnasmXLbW9UKUnjxo2zaCrXwxEYAABcxNdff60+ffro6tWrysvLk6+vr7777jtVrlxZfn5+OnXqlNUjugxehQQAgIuYMGGC+vbtq8uXL6tSpUrauXOnzp49q/bt22vOnDlWj+dSOAIDAICL8PHx0a5du9SkSRP5+PgoKSlJzZo1065duzRs2DDHhb3gCAwAAC6jQoUKjs9G8vPzU2pqqqQf38guLS3NytFcDhfxAgDgItq2bas9e/aoUaNG6tq1q2JjY/Xdd9/pww8/VMuWLa0ez6VwCgkAABeRnJysK1eu6NFHH1VmZqaefPJJ7dixQ40bN9Z7772nNm3aWD2iyyBgAABwET/88IOKiopUuXJlSdKZM2e0atUqNW/eXOHh4RZP51q4BgYAABfRr18/LVmyRJKUlZWljh07au7cuerfv78WLFhg8XSuhYABAMBF7Nu3T507d5YkrVixQv7+/jp79qyWLFmiuLg4i6dzLQQMAAAu4urVq6pWrZokacOGDRowYIDc3d3VsWNHnT171uLpXAsBAwCAi2jYsKFWr16ttLQ0rV+/Xj179pQkZWZmymazWTydayFgAABwEbGxsXrhhRdUt25dhYSEKDQ0VNKPR2Patm1r8XSuhVchAQDgQtLT03XhwgW1bt3a8aZ2u3fvls1mU9OmTS2eznUQMAAAwDicQgIAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAWCJbt26afz48b9p7ebNm+Xm5qasrKzfdZ9169bVvHnzftfPAOAaCBgAAGAcAgYAABiHgAFguQ8//FAdOnRQtWrVFBAQoCFDhigzM/O2ddu3b1dwcLAqVqyojh076uDBg077v/rqK3Xu3FmVKlVSUFCQxo0bp7y8vNJ6GABKEQEDwHLXr1/Xyy+/rP3792v16tU6c+aMhg8fftu6iRMn6o033tCePXtUs2ZN9e3bV9evX5cknTx5Ur169dLAgQN14MABLV++XF999ZWio6NL+dEAKA2eVg8AAE899ZTjn+vXr6+4uDg99NBDys3NVdWqVR37pk6dqj/+8Y+SpA8++EAPPvigVq1apUGDBmnmzJmKjIx0XBjcqFEjxcXFqWvXrlqwYIEqVqxYqo8JQMniCAwAy+3du1d9+/ZV7dq1Va1aNXXt2lWSlJqa6rTu1gfbSZKvr6+aNGmiI0eOSJL279+vxYsXq2rVqo5beHi4CgsLdfr06dJ7MABKBUdgAFgqLy9P4eHhCg8P19KlS1WzZk2lpqYqPDxcBQUFv/nn5Obm6plnntG4ceNu21e7du3iHBmACyBgAFjq6NGj+v777zVr1iwFBQVJkpKTk++4dufOnY4YuXz5so4fP65mzZpJktq1a6fDhw+rYcOGpTM4AEtxCgmApWrXri0vLy+9/fbbOnXqlD7//HO9/PLLd1w7Y8YMbdy4UQcPHtTw4cNVo0YN9e/fX5I0efJk7dixQ9HR0UpJSdG3336rf/zjH1zEC5RRBAwAS9WsWVOLFy/Wp59+qubNm2vWrFmaM2fOHdfOmjVLzz33nNq3b6/09HR98cUX8vLykiQFBwdry5YtOn78uDp37qy2bdsqNjZWgYGBpflwAJQSt6KioiKrhwAAALgXHIEBAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAY5/8B94OlH4vvLhoAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data['label'].value_counts().plot(kind=\"bar\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d90bfedf-e3fc-4a4c-aba1-8fe18e72b605",
   "metadata": {},
   "outputs": [],
   "source": [
    "g = data.groupby('label')\n",
    "data = pd.DataFrame(g.apply(lambda x: x.sample(g.size().min()).reset_index(drop=True)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0a9cd78d-efb6-4b2a-8385-d439bde5aeef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: xlabel='label'>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAHeCAYAAACFVzEAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA1U0lEQVR4nO3dfVhUdf7/8dcgAt4wIBqM7IJaut7fW8imdiMrKZkm32/rxpq1pK0LmlpmXFdSUps3mRlmkt2prW5ubbJlLspqiSWioJipYeYNbDaw+0UgMAFlfn/0c7ZJa3V3cOaDz8d1nevyfD7vM+d9nMIXZ86ZY3E4HA4BAAAYxMfTDQAAAFwuAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMM5lB5icnByNHj1a4eHhslgsyszMvKDm0KFDuuOOOxQUFKRWrVrp+uuvV3FxsXP+zJkzSkpKUtu2bdW6dWvFx8ertLTU5TWKi4sVFxenli1bKjQ0VLNmzdLZs2cv/wgBAECT43u5G9TU1Khv3776zW9+o3Hjxl0w/8UXX2jIkCFKTEzU3LlzZbVadeDAAQUEBDhrZsyYoffff19vvfWWgoKClJycrHHjxunjjz+WJJ07d05xcXGy2WzasWOHvvrqK91zzz1q3ry5nn766Uvqs6GhQSdPnlRgYKAsFsvlHiYAAPAAh8Ohr7/+WuHh4fLx+ZHzLI7/giTH+vXrXcZ++ctfOn7961//4DYVFRWO5s2bO9566y3n2KFDhxySHLm5uQ6Hw+HYuHGjw8fHx2G32501y5cvd1itVkdtbe0l9VZSUuKQxMLCwsLCwmLgUlJS8qP/zl/2GZgf09DQoPfff1+PPPKIYmNjtXfvXnXq1EkpKSkaO3asJKmgoED19fWKiYlxbtetWzdFRkYqNzdXgwcPVm5urnr37q2wsDBnTWxsrKZMmaIDBw6of//+F+y7trZWtbW1znXH/3/IdklJiaxWqzsPEwAANJKqqipFREQoMDDwR+vcGmDKyspUXV2t+fPn66mnntKCBQuUlZWlcePG6YMPPtBNN90ku90uPz8/BQcHu2wbFhYmu90uSbLb7S7h5fz8+bmLmTdvnubOnXvBuNVqJcAAAGCYf3f5h1vvQmpoaJAkjRkzRjNmzFC/fv306KOP6vbbb1dGRoY7d3WBlJQUVVZWOpeSkpJG3R8AAPActwaYdu3aydfXVz169HAZ7969u/MuJJvNprq6OlVUVLjUlJaWymazOWu+f1fS+fXzNd/n7+/vPNvCWRcAAJo2twYYPz8/XX/99SoqKnIZP3z4sDp06CBJGjhwoJo3b64tW7Y454uKilRcXKzo6GhJUnR0tPbv36+ysjJnTXZ2tqxW6wXhCAAAXH0u+xqY6upqHTlyxLl+7NgxFRYWKiQkRJGRkZo1a5Z++ctfatiwYbrllluUlZWl9957Tx9++KEkKSgoSImJiZo5c6ZCQkJktVo1depURUdHa/DgwZKkESNGqEePHpowYYIWLlwou92uxx57TElJSfL393fPkQMAAHNd0j3J3/HBBx9c9HaniRMnOmteffVVR+fOnR0BAQGOvn37OjIzM11e45tvvnH87ne/c7Rp08bRsmVLx5133un46quvXGqOHz/uGDlypKNFixaOdu3aOR566CFHfX39JfdZWVnpkOSorKy83EMEAAAecqn/flscjv9/v3ETU1VVpaCgIFVWVnI9DAAAhrjUf795FhIAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMM5lPwsJl6bjo+97ugWPOD4/ztMteATv99WF9/vqwvvtnTgDAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjHPZASYnJ0ejR49WeHi4LBaLMjMzf7D2t7/9rSwWi5YsWeIyXl5eroSEBFmtVgUHBysxMVHV1dUuNZ988omGDh2qgIAARUREaOHChZfbKgAAaKIuO8DU1NSob9++WrZs2Y/WrV+/Xjt37lR4ePgFcwkJCTpw4ICys7O1YcMG5eTkaPLkyc75qqoqjRgxQh06dFBBQYGeeeYZPfHEE1qxYsXltgsAAJog38vdYOTIkRo5cuSP1nz55ZeaOnWqNm3apLi4OJe5Q4cOKSsrS7t379agQYMkSUuXLtWoUaO0aNEihYeHa82aNaqrq9Nrr70mPz8/9ezZU4WFhVq8eLFL0AEAAFcnt18D09DQoAkTJmjWrFnq2bPnBfO5ubkKDg52hhdJiomJkY+Pj/Ly8pw1w4YNk5+fn7MmNjZWRUVFOnXq1EX3W1tbq6qqKpcFAAA0TW4PMAsWLJCvr6+mTZt20Xm73a7Q0FCXMV9fX4WEhMhutztrwsLCXGrOr5+v+b558+YpKCjIuURERPy3hwIAALyUWwNMQUGBnn/+ea1cuVIWi8WdL/1vpaSkqLKy0rmUlJRc0f0DAIArx60BZvv27SorK1NkZKR8fX3l6+urEydO6KGHHlLHjh0lSTabTWVlZS7bnT17VuXl5bLZbM6a0tJSl5rz6+drvs/f319Wq9VlAQAATZNbA8yECRP0ySefqLCw0LmEh4dr1qxZ2rRpkyQpOjpaFRUVKigocG63detWNTQ0KCoqylmTk5Oj+vp6Z012dra6du2qNm3auLNlAABgoMu+C6m6ulpHjhxxrh87dkyFhYUKCQlRZGSk2rZt61LfvHlz2Ww2de3aVZLUvXt33XbbbZo0aZIyMjJUX1+v5ORkjR8/3nnL9d133625c+cqMTFRs2fP1qeffqrnn39ezz333H9zrAAAoIm47ACTn5+vW265xbk+c+ZMSdLEiRO1cuXKS3qNNWvWKDk5WcOHD5ePj4/i4+OVnp7unA8KCtLmzZuVlJSkgQMHql27dkpNTeUWagAAIOk/CDA333yzHA7HJdcfP378grGQkBCtXbv2R7fr06ePtm/ffrntAQCAqwDPQgIAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcS47wOTk5Gj06NEKDw+XxWJRZmamc66+vl6zZ89W79691apVK4WHh+uee+7RyZMnXV6jvLxcCQkJslqtCg4OVmJioqqrq11qPvnkEw0dOlQBAQGKiIjQwoUL/7MjBAAATc5lB5iamhr17dtXy5Ytu2Du9OnT2rNnj+bMmaM9e/bonXfeUVFRke644w6XuoSEBB04cEDZ2dnasGGDcnJyNHnyZOd8VVWVRowYoQ4dOqigoEDPPPOMnnjiCa1YseI/OEQAANDU+F7uBiNHjtTIkSMvOhcUFKTs7GyXsRdeeEE33HCDiouLFRkZqUOHDikrK0u7d+/WoEGDJElLly7VqFGjtGjRIoWHh2vNmjWqq6vTa6+9Jj8/P/Xs2VOFhYVavHixS9ABAABXp0a/BqayslIWi0XBwcGSpNzcXAUHBzvDiyTFxMTIx8dHeXl5zpphw4bJz8/PWRMbG6uioiKdOnXqovupra1VVVWVywIAAJqmRg0wZ86c0ezZs/WrX/1KVqtVkmS32xUaGupS5+vrq5CQENntdmdNWFiYS8359fM13zdv3jwFBQU5l4iICHcfDgAA8BKNFmDq6+t11113yeFwaPny5Y21G6eUlBRVVlY6l5KSkkbfJwAA8IzLvgbmUpwPLydOnNDWrVudZ18kyWazqayszKX+7NmzKi8vl81mc9aUlpa61JxfP1/zff7+/vL393fnYQAAAC/l9jMw58PL559/rr/97W9q27aty3x0dLQqKipUUFDgHNu6dasaGhoUFRXlrMnJyVF9fb2zJjs7W127dlWbNm3c3TIAADDMZQeY6upqFRYWqrCwUJJ07NgxFRYWqri4WPX19fqf//kf5efna82aNTp37pzsdrvsdrvq6uokSd27d9dtt92mSZMmadeuXfr444+VnJys8ePHKzw8XJJ09913y8/PT4mJiTpw4IDWrVun559/XjNnznTfkQMAAGNd9kdI+fn5uuWWW5zr50PFxIkT9cQTT+jdd9+VJPXr189luw8++EA333yzJGnNmjVKTk7W8OHD5ePjo/j4eKWnpztrg4KCtHnzZiUlJWngwIFq166dUlNTuYUaAABI+g8CzM033yyHw/GD8z82d15ISIjWrl37ozV9+vTR9u3bL7c9AABwFeBZSAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwzmUHmJycHI0ePVrh4eGyWCzKzMx0mXc4HEpNTVX79u3VokULxcTE6PPPP3epKS8vV0JCgqxWq4KDg5WYmKjq6mqXmk8++URDhw5VQECAIiIitHDhwss/OgAA0CRddoCpqalR3759tWzZsovOL1y4UOnp6crIyFBeXp5atWql2NhYnTlzxlmTkJCgAwcOKDs7Wxs2bFBOTo4mT57snK+qqtKIESPUoUMHFRQU6JlnntETTzyhFStW/AeHCAAAmhrfy91g5MiRGjly5EXnHA6HlixZoscee0xjxoyRJK1evVphYWHKzMzU+PHjdejQIWVlZWn37t0aNGiQJGnp0qUaNWqUFi1apPDwcK1Zs0Z1dXV67bXX5Ofnp549e6qwsFCLFy92CToAAODq5NZrYI4dOya73a6YmBjnWFBQkKKiopSbmytJys3NVXBwsDO8SFJMTIx8fHyUl5fnrBk2bJj8/PycNbGxsSoqKtKpU6cuuu/a2lpVVVW5LAAAoGlya4Cx2+2SpLCwMJfxsLAw55zdbldoaKjLvK+vr0JCQlxqLvYa393H982bN09BQUHOJSIi4r8/IAAA4JWazF1IKSkpqqysdC4lJSWebgkAADQStwYYm80mSSotLXUZLy0tdc7ZbDaVlZW5zJ89e1bl5eUuNRd7je/u4/v8/f1ltVpdFgAA0DS5NcB06tRJNptNW7ZscY5VVVUpLy9P0dHRkqTo6GhVVFSooKDAWbN161Y1NDQoKirKWZOTk6P6+npnTXZ2trp27ao2bdq4s2UAAGCgyw4w1dXVKiwsVGFhoaRvL9wtLCxUcXGxLBaLpk+frqeeekrvvvuu9u/fr3vuuUfh4eEaO3asJKl79+667bbbNGnSJO3atUsff/yxkpOTNX78eIWHh0uS7r77bvn5+SkxMVEHDhzQunXr9Pzzz2vmzJluO3AAAGCuy76NOj8/X7fccotz/XyomDhxolauXKlHHnlENTU1mjx5sioqKjRkyBBlZWUpICDAuc2aNWuUnJys4cOHy8fHR/Hx8UpPT3fOBwUFafPmzUpKStLAgQPVrl07paamcgs1AACQ9B8EmJtvvlkOh+MH5y0Wi9LS0pSWlvaDNSEhIVq7du2P7qdPnz7avn375bYHAACuAk3mLiQAAHD1IMAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHHcHmDOnTunOXPmqFOnTmrRooWuu+46Pfnkk3I4HM4ah8Oh1NRUtW/fXi1atFBMTIw+//xzl9cpLy9XQkKCrFargoODlZiYqOrqane3CwAADOT2ALNgwQItX75cL7zwgg4dOqQFCxZo4cKFWrp0qbNm4cKFSk9PV0ZGhvLy8tSqVSvFxsbqzJkzzpqEhAQdOHBA2dnZ2rBhg3JycjR58mR3twsAAAzk6+4X3LFjh8aMGaO4uDhJUseOHfXHP/5Ru3btkvTt2ZclS5boscce05gxYyRJq1evVlhYmDIzMzV+/HgdOnRIWVlZ2r17twYNGiRJWrp0qUaNGqVFixYpPDzc3W0DAACDuP0MzM9//nNt2bJFhw8fliTt27dPH330kUaOHClJOnbsmOx2u2JiYpzbBAUFKSoqSrm5uZKk3NxcBQcHO8OLJMXExMjHx0d5eXkX3W9tba2qqqpcFgAA0DS5/QzMo48+qqqqKnXr1k3NmjXTuXPn9Pvf/14JCQmSJLvdLkkKCwtz2S4sLMw5Z7fbFRoa6tqor69CQkKcNd83b948zZ07192HAwAAvJDbz8D86U9/0po1a7R27Vrt2bNHq1at0qJFi7Rq1Sp378pFSkqKKisrnUtJSUmj7g8AAHiO28/AzJo1S48++qjGjx8vSerdu7dOnDihefPmaeLEibLZbJKk0tJStW/f3rldaWmp+vXrJ0my2WwqKytzed2zZ8+qvLzcuf33+fv7y9/f392HAwAAvJDbz8CcPn1aPj6uL9usWTM1NDRIkjp16iSbzaYtW7Y456uqqpSXl6fo6GhJUnR0tCoqKlRQUOCs2bp1qxoaGhQVFeXulgEAgGHcfgZm9OjR+v3vf6/IyEj17NlTe/fu1eLFi/Wb3/xGkmSxWDR9+nQ99dRT6tKlizp16qQ5c+YoPDxcY8eOlSR1795dt912myZNmqSMjAzV19crOTlZ48eP5w4kAADg/gCzdOlSzZkzR7/73e9UVlam8PBwPfDAA0pNTXXWPPLII6qpqdHkyZNVUVGhIUOGKCsrSwEBAc6aNWvWKDk5WcOHD5ePj4/i4+OVnp7u7nYBAICB3B5gAgMDtWTJEi1ZsuQHaywWi9LS0pSWlvaDNSEhIVq7dq272wMAAE0Az0ICAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEaJcB8+eWX+vWvf622bduqRYsW6t27t/Lz853zDodDqampat++vVq0aKGYmBh9/vnnLq9RXl6uhIQEWa1WBQcHKzExUdXV1Y3RLgAAMIzbA8ypU6d04403qnnz5vrrX/+qgwcP6tlnn1WbNm2cNQsXLlR6eroyMjKUl5enVq1aKTY2VmfOnHHWJCQk6MCBA8rOztaGDRuUk5OjyZMnu7tdAABgIF93v+CCBQsUERGh119/3TnWqVMn558dDoeWLFmixx57TGPGjJEkrV69WmFhYcrMzNT48eN16NAhZWVlaffu3Ro0aJAkaenSpRo1apQWLVqk8PBwd7cNAAAM4vYzMO+++64GDRqk//3f/1VoaKj69++vl19+2Tl/7Ngx2e12xcTEOMeCgoIUFRWl3NxcSVJubq6Cg4Od4UWSYmJi5OPjo7y8vIvut7a2VlVVVS4LAABomtweYI4eParly5erS5cu2rRpk6ZMmaJp06Zp1apVkiS73S5JCgsLc9kuLCzMOWe32xUaGuoy7+vrq5CQEGfN982bN09BQUHOJSIiwt2HBgAAvITbA0xDQ4MGDBigp59+Wv3799fkyZM1adIkZWRkuHtXLlJSUlRZWelcSkpKGnV/AADAc9weYNq3b68ePXq4jHXv3l3FxcWSJJvNJkkqLS11qSktLXXO2Ww2lZWVucyfPXtW5eXlzprv8/f3l9VqdVkAAEDT5PYAc+ONN6qoqMhl7PDhw+rQoYOkby/otdls2rJli3O+qqpKeXl5io6OliRFR0eroqJCBQUFzpqtW7eqoaFBUVFR7m4ZAAAYxu13Ic2YMUM///nP9fTTT+uuu+7Srl27tGLFCq1YsUKSZLFYNH36dD311FPq0qWLOnXqpDlz5ig8PFxjx46V9O0Zm9tuu8350VN9fb2Sk5M1fvx47kACAADuDzDXX3+91q9fr5SUFKWlpalTp05asmSJEhISnDWPPPKIampqNHnyZFVUVGjIkCHKyspSQECAs2bNmjVKTk7W8OHD5ePjo/j4eKWnp7u7XQAAYCC3BxhJuv3223X77bf/4LzFYlFaWprS0tJ+sCYkJERr165tjPYAAIDheBYSAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIzT6AFm/vz5slgsmj59unPszJkzSkpKUtu2bdW6dWvFx8ertLTUZbvi4mLFxcWpZcuWCg0N1axZs3T27NnGbhcAABigUQPM7t279dJLL6lPnz4u4zNmzNB7772nt956S9u2bdPJkyc1btw45/y5c+cUFxenuro67dixQ6tWrdLKlSuVmpramO0CAABDNFqAqa6uVkJCgl5++WW1adPGOV5ZWalXX31Vixcv1q233qqBAwfq9ddf144dO7Rz505J0ubNm3Xw4EH94Q9/UL9+/TRy5Eg9+eSTWrZsmerq6hqrZQAAYIhGCzBJSUmKi4tTTEyMy3hBQYHq6+tdxrt166bIyEjl5uZKknJzc9W7d2+FhYU5a2JjY1VVVaUDBw5cdH+1tbWqqqpyWQAAQNPk2xgv+uabb2rPnj3avXv3BXN2u11+fn4KDg52GQ8LC5PdbnfWfDe8nJ8/P3cx8+bN09y5c93QPQAA8HZuPwNTUlKiBx98UGvWrFFAQIC7X/4HpaSkqLKy0rmUlJRcsX0DAIAry+0BpqCgQGVlZRowYIB8fX3l6+urbdu2KT09Xb6+vgoLC1NdXZ0qKipctistLZXNZpMk2Wy2C+5KOr9+vub7/P39ZbVaXRYAANA0uT3ADB8+XPv371dhYaFzGTRokBISEpx/bt68ubZs2eLcpqioSMXFxYqOjpYkRUdHa//+/SorK3PWZGdny2q1qkePHu5uGQAAGMbt18AEBgaqV69eLmOtWrVS27ZtneOJiYmaOXOmQkJCZLVaNXXqVEVHR2vw4MGSpBEjRqhHjx6aMGGCFi5cKLvdrscee0xJSUny9/d3d8sAAMAwjXIR77/z3HPPycfHR/Hx8aqtrVVsbKxefPFF53yzZs20YcMGTZkyRdHR0WrVqpUmTpyotLQ0T7QLAAC8zBUJMB9++KHLekBAgJYtW6Zly5b94DYdOnTQxo0bG7kzAABgIp6FBAAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGcXuAmTdvnq6//noFBgYqNDRUY8eOVVFRkUvNmTNnlJSUpLZt26p169aKj49XaWmpS01xcbHi4uLUsmVLhYaGatasWTp79qy72wUAAAZye4DZtm2bkpKStHPnTmVnZ6u+vl4jRoxQTU2Ns2bGjBl677339NZbb2nbtm06efKkxo0b55w/d+6c4uLiVFdXpx07dmjVqlVauXKlUlNT3d0uAAAwkK+7XzArK8tlfeXKlQoNDVVBQYGGDRumyspKvfrqq1q7dq1uvfVWSdLrr7+u7t27a+fOnRo8eLA2b96sgwcP6m9/+5vCwsLUr18/Pfnkk5o9e7aeeOIJ+fn5ubttAABgkEa/BqayslKSFBISIkkqKChQfX29YmJinDXdunVTZGSkcnNzJUm5ubnq3bu3wsLCnDWxsbGqqqrSgQMHGrtlAADg5dx+Bua7GhoaNH36dN14443q1auXJMlut8vPz0/BwcEutWFhYbLb7c6a74aX8/Pn5y6mtrZWtbW1zvWqqip3HQYAAPAyjXoGJikpSZ9++qnefPPNxtyNpG8vHg4KCnIuERERjb5PAADgGY0WYJKTk7VhwwZ98MEH+ulPf+oct9lsqqurU0VFhUt9aWmpbDabs+b7dyWdXz9f830pKSmqrKx0LiUlJW48GgAA4E3cHmAcDoeSk5O1fv16bd26VZ06dXKZHzhwoJo3b64tW7Y4x4qKilRcXKzo6GhJUnR0tPbv36+ysjJnTXZ2tqxWq3r06HHR/fr7+8tqtbosAACgaXL7NTBJSUlau3at/vKXvygwMNB5zUpQUJBatGihoKAgJSYmaubMmQoJCZHVatXUqVMVHR2twYMHS5JGjBihHj16aMKECVq4cKHsdrsee+wxJSUlyd/f390tAwAAw7g9wCxfvlySdPPNN7uMv/7667r33nslSc8995x8fHwUHx+v2tpaxcbG6sUXX3TWNmvWTBs2bNCUKVMUHR2tVq1aaeLEiUpLS3N3uwAAwEBuDzAOh+Pf1gQEBGjZsmVatmzZD9Z06NBBGzdudGdrAACgieBZSAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwjlcHmGXLlqljx44KCAhQVFSUdu3a5emWAACAF/DaALNu3TrNnDlTjz/+uPbs2aO+ffsqNjZWZWVlnm4NAAB4mNcGmMWLF2vSpEm677771KNHD2VkZKhly5Z67bXXPN0aAADwMK8MMHV1dSooKFBMTIxzzMfHRzExMcrNzfVgZwAAwBv4erqBi/nnP/+pc+fOKSwszGU8LCxMn3322UW3qa2tVW1trXO9srJSklRVVdV4jf6IhtrTHtmvp3nq79vTeL+vLrzfVxfeb8/s1+Fw/GidVwaY/8S8efM0d+7cC8YjIiI80M3VK2iJpzvAlcT7fXXh/b66ePr9/vrrrxUUFPSD814ZYNq1a6dmzZqptLTUZby0tFQ2m+2i26SkpGjmzJnO9YaGBpWXl6tt27ayWCyN2q83qaqqUkREhEpKSmS1Wj3dDhoZ7/fVhff76nK1vt8Oh0Nff/21wsPDf7TOKwOMn5+fBg4cqC1btmjs2LGSvg0kW7ZsUXJy8kW38ff3l7+/v8tYcHBwI3fqvaxW61X1H/zVjvf76sL7fXW5Gt/vHzvzcp5XBhhJmjlzpiZOnKhBgwbphhtu0JIlS1RTU6P77rvP060BAAAP89oA88tf/lL/+Mc/lJqaKrvdrn79+ikrK+uCC3sBAMDVx2sDjCQlJyf/4EdGuDh/f389/vjjF3ychqaJ9/vqwvt9deH9/nEWx7+7TwkAAMDLeOUX2QEAAPwYAgwAADAOAQYAABiHAAMAXuro0aOebgHwWgQYw509e1ZpaWn6+9//7ulW0Mjq6+t13XXX6dChQ55uBVdI586ddcstt+gPf/iDzpw54+l2AK9CgDGcr6+vnnnmGZ09e9bTraCRNW/enH/ErjJ79uxRnz59NHPmTNlsNj3wwAPatWuXp9tCI/rmm290+vS/Hh554sQJLVmyRJs3b/ZgV96JANME3Hrrrdq2bZun28AVkJSUpAULFhBYrxL9+vXT888/r5MnT+q1117TV199pSFDhqhXr15avHix/vGPf3i6RbjZmDFjtHr1aklSRUWFoqKi9Oyzz2rMmDFavny5h7vzLnwPTBOQkZGhuXPnKiEhQQMHDlSrVq1c5u+44w4PdQZ3u/POO7Vlyxa1bt1avXv3vuC9fueddzzUGa6E2tpavfjii0pJSVFdXZ38/Px01113acGCBWrfvr2n24MbtGvXTtu2bVPPnj31yiuvaOnSpdq7d6/+/Oc/KzU1lY+Qv4MA0wT4+PzwiTSLxaJz585dwW7QmP7ds8Bef/31K9QJrqT8/Hy99tprevPNN9WqVStNnDhRiYmJ+vvf/665c+eqqqqKj5aaiJYtW+qzzz5TZGSk7rrrLvXs2VOPP/64SkpK1LVrV5ePl652BBgA8FKLFy/W66+/rqKiIo0aNUr333+/Ro0a5fJLy9///nd17NiRjxWbiD59+uj+++/XnXfeqV69eikrK0vR0dEqKChQXFyc7Ha7p1v0GlwD08RwkSfQdCxfvlx33323Tpw4oczMTN1+++0XnHENDQ3Vq6++6qEO4W6pqal6+OGH1bFjR0VFRSk6OlqStHnzZvXv39/D3XkXzsA0AefOndPTTz+tjIwMlZaW6vDhw7r22ms1Z84cdezYUYmJiZ5uEW709ttv609/+pOKi4tVV1fnMrdnzx4PdQXAXex2u7766iv17dvXGVh37dolq9Wqbt26ebg778EZmCbg97//vVauXKmFCxfKz8/POd6rVy+98sorHuwM7paenq777rtPYWFh2rt3r2644Qa1bdtWR48e1ciRIz3dHhpBRUWFnn32Wd1///26//779dxzz6mystLTbaER2Ww29e/fXz4+PqqqqlJmZqYCAwMJL9/DGZgmoHPnznrppZc0fPhwBQYGat++fbr22mv12WefKTo6WqdOnfJ0i3CTbt266fHHH9evfvUrl/c6NTVV5eXleuGFFzzdItwoPz9fsbGxatGihW644QZJ0u7du/XNN99o8+bNGjBggIc7hLvdddddGjZsmJKTk/XNN9+ob9++On78uBwOh958803Fx8d7ukWvwRmYJuDLL79U586dLxhvaGhQfX29BzpCYykuLtbPf/5zSVKLFi309ddfS5ImTJigP/7xj55sDY1gxowZuuOOO3T8+HG98847euedd3Ts2DHdfvvtmj59uqfbQyPIycnR0KFDJUnr16+Xw+FQRUWF0tPT9dRTT3m4O+9CgGkCevTooe3bt18w/vbbb3PRVxNjs9lUXl4uSYqMjNTOnTslSceOHRMnU5ue/Px8zZ49W76+vs4xX19fPfLII8rPz/dgZ2gslZWVCgkJkSRlZWUpPj5eLVu2VFxcnD7//HMPd+ddfP99CbxdamqqJk6cqC+//FINDQ165513VFRUpNWrV2vDhg2ebg9udOutt+rdd99V//79dd9992nGjBl6++23lZ+fr3Hjxnm6PbiZ1WpVcXHxBdc+lJSUKDAw0ENdoTFFREQoNzdXISEhysrK0ptvvilJOnXqlAICAjzcnXfhGpgmYvv27UpLS9O+fftUXV2tAQMGKDU1VSNGjPB0a3CjhoYGNTQ0OH8jf/PNN7Vjxw516dJFDzzwgMtF3DDftGnTtH79ei1atMj50eHHH3+sWbNmKT4+XkuWLPFsg3C7F198UQ8++KBat26tyMhI7d27Vz4+Plq6dKneeecdffDBB55u0WsQYADAS9XV1WnWrFnKyMjQ2bNn5XA45OfnpylTpmj+/Pny9/f3dItoBPn5+SopKdEvfvELtW7dWpL0/vvvKzg4WDfeeKOHu/MeBBjAMNu3b9dLL72kL774Qm+//bZ+8pOf6I033lCnTp00ZMgQT7eHRnD69Gl98cUXkqTrrrtOLVu29HBHaGx1dXU6duyYrrvuOpdroPAv/K00AW3atJHFYrlg3GKxKCAgQJ07d9a99977b5+jA+/35z//WRMmTFBCQoL27t2r2tpaSd9e+Pf0009r48aNHu4Q/61x48Zp5cqVslqt//a6ptatW6tnz5767W9/q6CgoCvUIRrT6dOnNXXqVK1atUqSnF9MOnXqVP3kJz/Ro48+6uEOvQd3ITUBqamp8vHxUVxcnObOnau5c+cqLi5OPj4+SkpK0s9+9jNNmTJFL7/8sqdbxX/pqaeeUkZGhl5++WU1b97cOX7jjTfyLbxNRFBQkPMXkqCgoB9dzp49q4yMDE2YMMHDXcNdUlJStG/fPn344YcuF+3GxMRo3bp1HuzM+/ARUhMQHx+vX/ziF/rtb3/rMv7SSy9p8+bN+vOf/6ylS5dqxYoV2r9/v4e6hDu0bNlSBw8eVMeOHV2+yO7o0aPq0aMHz8K6Ch08eFDXX3+9ampqPN0K3KBDhw5at26dBg8e7PL/+JEjRzRgwABVVVV5ukWvwRmYJmDTpk2KiYm5YHz48OHatGmTJGnUqFE6evTolW4Nbmaz2XTkyJELxj/66CNde+21HugInta1a1ft2LHD023ATf7xj38oNDT0gvGampqLXipwNSPANAEhISF67733Lhh/7733nF+IVFNTw/dGNAGTJk3Sgw8+qLy8PFksFp08eVJr1qzRww8/rClTpni6PXhAs2bN1LdvX0+3ATcZNGiQ3n//fef6+dDyyiuvOJ9MjW9xEW8TMGfOHE2ZMkUffPCBy/NSNm7cqIyMDElSdna2brrpJk+2if/QJ598ol69esnHx0cpKSlqaGjQ8OHDdfr0aQ0bNkz+/v56+OGHNXXqVE+3CuC/9PTTT2vkyJE6ePCgzp49q+eff14HDx7Ujh07tG3bNk+351W4BqaJ+Pjjj/XCCy+oqKhI0renladOner88iuYq1mzZvrqq68UGhqqa6+9Vrt371ZgYKCOHDmi6upq9ejRw/ldEQDM98UXX2j+/PkuX0w6e/Zs9e7d29OteRUCDODl2rZtq40bNyoqKko+Pj4qLS3VNddc4+m2AMCj+AipiWhoaNCRI0dUVlamhoYGl7lhw4Z5qCu4Q3x8vG666Sa1b99eFotFgwYNUrNmzS5ay4XagPn4eX5pCDBNwM6dO3X33XfrxIkTFzyR2GKx6Ny5cx7qDO6wYsUKjRs3TkeOHNG0adM0adIkLsgGmih+nl86PkJqAvr166ef/exnmjt3rvO39O/iGzqbjvvuu0/p6ekEGKCJ4uf5pSPANAGtWrXSvn371LlzZ0+3AgD4L/Dz/NLxPTBNQFRU1EW/3AwAYBZ+nl86roFpAqZOnaqHHnpIdrtdvXv3dnlGjiT16dPHQ50BAC4HP88vHR8hNQE+Pj98Io2LvgDAHBf7eW6xWORwOPh5/j2cgWkCjh075ukWAABuwM/zS8cZmCbk4MGDKi4uVl1dnXPMYrFo9OjRHuwKAAD34wxME3D06FHdeeed2r9/v/NUo/Svh4BxyhEAvNe77757ybV33HFHI3ZiFs7ANAGjR49Ws2bN9Morr6hTp07Ky8tTeXm5HnroIS1atEhDhw71dIsAgB/w/etevvuL6Pn18/iF9F+4jboJyM3NVVpamtq1aycfHx81a9ZMQ4YM0bx58zRt2jRPtwcA+BENDQ3OZfPmzerXr5/++te/qqKiQhUVFdq4caMGDBigrKwsT7fqVfgIqQk4d+6c85tZ27Vrp5MnT6pr167q0KGD8+nUAADvN336dGVkZGjIkCHOsdjYWLVs2VKTJ0/WoUOHPNiddyHANAG9evXSvn371KlTJ0VFRWnhwoXy8/PTihUrdO2113q6PQDAJfriiy8UHBx8wXhQUJCOHz9+xfvxZlwD0wRs2rRJNTU1zgf+3X777Tp8+LDatm2rdevW6dZbb/V0iwCASzBs2DAFBATojTfeUFhYmCSptLRU99xzj86cOaNt27Z5uEPvQYBposrLy9WmTZsLHgQGAPBeR44c0Z133qnDhw8rIiJCklRSUqIuXbooMzOTZyR9BwEGAAAv4nA4lJ2drc8++0yS1L17d8XExPAL6fcQYAAAgHG4iBcAAC9SU1Ojbdu2XfDN6pL4aozv4AwMAABeYu/evRo1apROnz6tmpoahYSE6J///Kdatmyp0NBQHT161NMteg2+yA4AAC8xY8YMjR49WqdOnVKLFi20c+dOnThxQgMHDtSiRYs83Z5X4QwMAABeIjg4WHl5eeratauCg4OVm5ur7t27Ky8vTxMnTnRe2AvOwAAA4DWaN2/ufDZSaGioiouLJX37RXYlJSWebM3rcBEvAABeon///tq9e7e6dOmim266SampqfrnP/+pN954Q7169fJ0e16Fj5AAAPAS+fn5+vrrr3XLLbeorKxM99xzj3bs2KGf/exneuWVV9SvXz9Pt+g1CDAAAHiJb775Rg6HQy1btpQkHT9+XOvXr1ePHj0UGxvr4e68C9fAAADgJcaMGaPVq1dLkioqKjR48GAtXrxYY8eO1fLlyz3cnXchwAAA4CX27NmjoUOHSpLefvtthYWF6cSJE1q9erXS09M93J13IcAAAOAlTp8+rcDAQEnS5s2bNW7cOPn4+Gjw4ME6ceKEh7vzLgQYAAC8ROfOnZWZmamSkhJt2rRJI0aMkCSVlZXJarV6uDvvQoABAMBLpKam6uGHH1bHjh0VFRWl6OhoSd+ejenfv7+Hu/Mu3IUEAIAXsdvt+uqrr9S3b1/nl9rt2rVLVqtV3bp183B33oMAAwAAjMNHSAAAwDgEGAAAYBwCDAAAMA4BBoBH3HzzzZo+ffol1X744YeyWCyqqKj4r/bZsWNHLVmy5L96DQDegQADAACMQ4ABAADGIcAA8Lg33nhDgwYNUmBgoGw2m+6++26VlZVdUPfxxx+rT58+CggI0ODBg/Xpp5+6zH/00UcaOnSoWrRooYiICE2bNk01NTVX6jAAXEEEGAAeV19fryeffFL79u1TZmamjh8/rnvvvfeCulmzZunZZ5/V7t27dc0112j06NGqr6+XJH3xxRe67bbbFB8fr08++UTr1q3TRx99pOTk5Ct8NACuBF9PNwAAv/nNb5x/vvbaa5Wenq7rr79e1dXVat26tXPu8ccf1y9+8QtJ0qpVq/TTn/5U69ev11133aV58+YpISHBeWFwly5dlJ6erptuuknLly9XQEDAFT0mAI2LMzAAPK6goECjR49WZGSkAgMDddNNN0mSiouLXerOPxdGkkJCQtS1a1cdOnRIkrRv3z6tXLlSrVu3di6xsbFqaGjQsWPHrtzBALgiOAMDwKNqamoUGxur2NhYrVmzRtdcc42Ki4sVGxururq6S36d6upqPfDAA5o2bdoFc5GRke5sGYAXIMAA8KjPPvtM//d//6f58+crIiJCkpSfn3/R2p07dzrDyKlTp3T48GF1795dkjRgwAAdPHhQnTt3vjKNA/AoPkIC4FGRkZHy8/PT0qVLdfToUb377rt68sknL1qblpamLVu26NNPP9W9996rdu3aaezYsZKk2bNna8eOHUpOTlZhYaE+//xz/eUvf+EiXqCJIsAA8KhrrrlGK1eu1FtvvaUePXpo/vz5WrRo0UVr58+frwcffFADBw6U3W7Xe++9Jz8/P0lSnz59tG3bNh0+fFhDhw5V//79lZqaqvDw8Ct5OACuEIvD4XB4ugkAAIDLwRkYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIzz/wCR5xrs1hunFwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data['label'].value_counts().plot(kind=\"bar\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9d973270-b3f6-4429-8ff1-0fe3119de4cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "data['label_int'] = LabelEncoder().fit_transform(data['label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1bb26f30-8271-4cc3-aa65-f9b745bbc483",
   "metadata": {},
   "outputs": [],
   "source": [
    "NUM_LABELS = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "007b6f9b-513a-4a16-8bce-6beda9159fed",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_split, test_split = train_test_split(data, train_size = 0.8)\n",
    "train_split, val_split = train_test_split(train_split, train_size = 0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "5d0ec884-52e4-4bff-b66a-ae9359c7a37f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4414\n",
      "1227\n",
      "491\n"
     ]
    }
   ],
   "source": [
    "print(len(train_split))\n",
    "print(len(test_split))\n",
    "print(len(val_split))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "1c28729d-67f6-451e-b57d-d488308a6b84",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.DataFrame({\n",
    "    \"label\": train_split.label_int.values,\n",
    "    \"text\": train_split.text_clean.values\n",
    "})\n",
    "\n",
    "test_df = pd.DataFrame({\n",
    "    \"label\": test_split.label_int.values,\n",
    "    \"text\": test_split.text_clean.values\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "a45fbde5-2b92-4117-a019-d0cee53942db",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = datasets.Dataset.from_dict(train_df)\n",
    "test_df = datasets.Dataset.from_dict(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "886403ea-5cbd-4ffb-93c7-a9df0affb2c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_dict = datasets.DatasetDict({\"train\":train_df, \"test\":test_df})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "a5a5229f-056d-4de8-9924-6b333cabe90a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['label', 'text'],\n",
       "        num_rows: 4414\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['label', 'text'],\n",
       "        num_rows: 1227\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ac248b3-4c91-4a73-9109-a1a32fd4e810",
   "metadata": {},
   "source": [
    "## Create embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "8f2eb996-abea-43be-9b91-726242c594a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = XLNetTokenizer.from_pretrained(\"xlnet-base-cased\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "ec11bb76-9aee-420e-bed7-9d06c5d4d395",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize_function(examples):\n",
    "    return tokenizer(examples[\"text\"], padding = \"max_length\", max_length = 128, truncation=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "dadb49e0-2e90-4fb8-b2be-a0eb9203de08",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Map: 100%|████████████████████████████████████████████████████████████████| 4414/4414 [00:02<00:00, 1866.56 examples/s]\n",
      "Map: 100%|████████████████████████████████████████████████████████████████| 1227/1227 [00:00<00:00, 2139.75 examples/s]\n"
     ]
    }
   ],
   "source": [
    "tokenized_datasets = dataset_dict.map(tokenize_function, batched=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "a10b75fc-b5cd-4ced-9cf5-7d535690df50",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['label', 'text', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
       "        num_rows: 4414\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['label', 'text', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
       "        num_rows: 1227\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "a630aae8-e818-482d-89c3-5f099864fa19",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unmatched party specialist /co  #serious #job #titles\n"
     ]
    }
   ],
   "source": [
    "print(tokenized_datasets['train']['text'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "fbd33c34-6b61-4787-b0e4-4ef230d86d3c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 31797, 489, 6273, 17, 167, 721, 17, 7967, 18968, 17, 7967, 25308, 17, 7967, 25612, 23, 4, 3]\n"
     ]
    }
   ],
   "source": [
    "print(tokenized_datasets['train']['input_ids'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "44fbee45-7e9f-47e5-a983-8a1e74ab04d4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<pad>'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "ec6a98f2-90ba-42df-ae35-d2face9d809d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]\n"
     ]
    }
   ],
   "source": [
    "print(tokenized_datasets['train']['token_type_ids'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "e354345c-d392-4727-84d2-c85d770d777d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n"
     ]
    }
   ],
   "source": [
    "print(tokenized_datasets['train']['attention_mask'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "3f626c47-8dcb-4729-8509-51a6961ad210",
   "metadata": {},
   "outputs": [],
   "source": [
    "small_train_dataset = tokenized_datasets[\"train\"].shuffle(seed=42).select(range(100))\n",
    "small_eval_dataset = tokenized_datasets[\"test\"].shuffle(seed=42).select(range(100))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af64b777-3499-444c-af1e-efa8a7980efe",
   "metadata": {},
   "source": [
    "## Fine tune our model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "92715b23-7a48-4a60-8d95-c1154d5978d3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of XLNetForSequenceClassification were not initialized from the model checkpoint at xlnet-base-cased and are newly initialized: ['logits_proj.bias', 'sequence_summary.summary.bias', 'logits_proj.weight', 'sequence_summary.summary.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "model = XLNetForSequenceClassification.from_pretrained('xlnet-base-cased', \n",
    "                                                       num_labels=NUM_LABELS, \n",
    "                                                       id2label={0: 'anger', 1: 'fear', 2: 'joy', 3: 'sadness'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "8e67a209-5dab-4815-9139-6920916d17e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "metric = evaluate.load(\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "64d861b6-4927-4503-8513-57fa0d3c50c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_metrics(eval_pred):\n",
    "    logits, labels = eval_pred\n",
    "    predictions = np.argmax(logits, axis=-1)\n",
    "    return metric.compute(predictions=predictions, references=labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "6fd259f4-253d-4051-aa04-8638ade95d14",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_args = TrainingArguments(output_dir=\"test_trainer\", evaluation_strategy=\"epoch\", num_train_epochs=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "ea9813f0-aea7-474d-92d6-a5d56185a575",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer = Trainer(\n",
    "    model=model, \n",
    "    args=training_args,\n",
    "    train_dataset=small_train_dataset,\n",
    "    eval_dataset=small_eval_dataset,\n",
    "    compute_metrics=compute_metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "bf0bbb82-d7b3-4670-b9cd-fded094070d4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='39' max='39' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [39/39 09:50, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.403604</td>\n",
       "      <td>0.270000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.399921</td>\n",
       "      <td>0.170000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.397637</td>\n",
       "      <td>0.190000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=39, training_loss=1.3937044388208635, metrics={'train_runtime': 603.5068, 'train_samples_per_second': 0.497, 'train_steps_per_second': 0.065, 'total_flos': 21366375321600.0, 'train_loss': 1.3937044388208635, 'epoch': 3.0})"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc332902-982f-46f4-8803-24cc6300322d",
   "metadata": {},
   "source": [
    "## Evaluate model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "1c847048-ef7f-437e-a9c3-67379595ea5b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'eval_loss': 1.3976367712020874,\n",
       " 'eval_accuracy': 0.19,\n",
       " 'eval_runtime': 49.9493,\n",
       " 'eval_samples_per_second': 2.002,\n",
       " 'eval_steps_per_second': 0.26,\n",
       " 'epoch': 3.0}"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "66c6417a-8b84-437c-b42c-34155ffb2753",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_pretrained(\"fine_tuned_model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "ccb7e982-af4d-4e71-ac2a-d671a17f49a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "fine_tuned_model = XLNetForSequenceClassification.from_pretrained(\"fine_tuned_model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "d9332c44-901f-4ee5-b920-27dc1d5ebf84",
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = pipeline(\"text-classification\", fine_tuned_model, tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "ec000c59-047e-4f4b-bbd3-c925b6106cf0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "come to the funeral tomorrow at 12 to mourn the death of my gpa\n",
      "[{'label': 'joy', 'score': 0.2925475537776947}, {'label': 'anger', 'score': 0.2373458594083786}, {'label': 'sadness', 'score': 0.23712345957756042}, {'label': 'fear', 'score': 0.23298314213752747}]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lnewb\\AppData\\Local\\Temp\\ipykernel_3276\\3116462782.py:2: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  print(val_split['text_clean'][rand_int])\n",
      "C:\\Users\\lnewb\\AppData\\Local\\Temp\\ipykernel_3276\\3116462782.py:3: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
      "  answer = clf(val_split['text_clean'][rand_int], top_k=None)\n"
     ]
    }
   ],
   "source": [
    "rand_int = random.randint(0, len(val_split))\n",
    "print(val_split['text_clean'][rand_int])\n",
    "answer = clf(val_split['text_clean'][rand_int], top_k=None)\n",
    "print(answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "954a0fa7-04cd-40bb-9761-ef2a59d357c2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd79f7a5-15a2-4372-897b-583153b060a4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8f1ecf2-6935-44c2-92f0-d8706ee2f70b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00c9c195-9882-49f0-9f5a-8c87cdf94536",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65ddb1a6-472a-47a0-a87d-894190452bc4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e97f16e-d442-4dbb-b385-69a8a17d5bb3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a08a9686-1a4b-4760-98de-8a0a829fadb1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5e1f906-b157-417e-8ef0-609d674ca018",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9c4aeca-de55-440b-a27c-05a611deab82",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b4bd22b-f53b-4ec1-87c1-166c2b9074ab",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0510b832-3194-4ba0-ab14-f6fd10bd911e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da96b67e-5398-48fa-8da1-432b8af65ed5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90e76acc-34e2-46a5-8d03-091ddb63ac2f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27993520-85fd-41e5-a843-d2ce3490d3f9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28906876-2267-4c6d-8662-fa40340f7ce0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "599ff1e2-46f8-492c-9ea4-06276042c5b2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0193b98-f6f6-4b53-b0ca-2e82ee0956f4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c63632b-b811-4622-ae2c-d2771c91d367",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7da99713-f5db-4cc7-bdb4-b06ce4f78eae",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6d027a0-2319-436e-8b5b-3c3c8729194d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6904bfe3-94d6-496e-aefa-4a23d9987ab0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fcb8cb89-9439-40d3-bea6-3809b780ac6b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1d603e7-7b0e-4533-9c06-2607ad920504",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1cbeeb51-ba2c-4c1f-b8dd-4eb55b319634",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b8867b3-fadf-4389-b441-1a498d20ce4b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}