ALBERT_for_SNLI.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ALBERT for SNLI.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Interpause/pseudo-text/blob/master/ALBERT_for_SNLI.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3t5TOnpXhu5U",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "outputId": "0850de6b-4aea-42f4-bd71-acc93be4ef96"
      },
      "source": [
        "%matplotlib inline\n",
        "%load_ext google.colab.data_table\n",
        "%tensorflow_version 2.x\n",
        "from google.colab import drive #drive.flush_and_unmount()\n",
        "from os import chdir\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "drive.mount('/content/drive')\n",
        "chdir('/content/drive/My Drive/CSIT Internship/code')\n",
        "\n",
        "import torch\n",
        "from torch import nn\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "#!pip install transformers"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TensorFlow 2.x selected.\n",
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gu1Q1Sprkfs1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import json\n",
        "data = []\n",
        "with open('./snli_1.0/snli_1.0_train.jsonl','r') as f:\n",
        "    for line in f.readlines():\n",
        "        data.append(json.loads(line))\n",
        "df = pd.DataFrame(data)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5lx2ONgZpxbJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import itertools\n",
        "import operator\n",
        "def most_common(L):\n",
        "  # get an iterable of (item, iterable) pairs\n",
        "  SL = sorted((x, i) for i, x in enumerate(L))\n",
        "  # print 'SL:', SL\n",
        "  groups = itertools.groupby(SL, key=operator.itemgetter(0))\n",
        "  # auxiliary function to get \"quality\" for an item\n",
        "  def _auxfun(g):\n",
        "    item, iterable = g\n",
        "    count = 0\n",
        "    min_index = len(L)\n",
        "    for _, where in iterable:\n",
        "      count += 1\n",
        "      min_index = min(min_index, where)\n",
        "    # print 'item %r, count %r, minind %r' % (item, count, min_index)\n",
        "    return count, -min_index\n",
        "  # pick the highest-count/earliest item\n",
        "  return max(groups, key=_auxfun)[0]\n",
        "\n",
        "def fixEmpty(r):\n",
        "    r.gold_label = most_common(r.annotator_labels)\n",
        "    return r\n",
        "df[df.gold_label == '-'] = df[df.gold_label == '-'].apply(fixEmpty,axis=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8OFrMlcG4HAB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from transformers import AlbertConfig, AlbertForSequenceClassification, AlbertTokenizer,AdamW\n",
        "pretrained = \"albert-base-v2\"\n",
        "tokenizer = AlbertTokenizer.from_pretrained(pretrained)\n",
        "#config = AlbertConfig.from_pretrained(pretrained)\n",
        "model = AlbertForSequenceClassification.from_pretrained(pretrained)\n",
        "model._modules['classifier'] = nn.Linear(in_features=768,out_features=3,bias=True)\n",
        "model.config.num_labels = 3\n",
        "model.num_labels = 3\n",
        "\n",
        "classlist = {'contradiction':0,\n",
        "             'entailment':1,\n",
        "             'neutral':2}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W10SKQ9AIz7A",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "options = {\n",
        "    'max_length':                   512,            #causes overflow\n",
        "    'stride':                       0,              #how many tokens before to start from in a overflow\n",
        "    'add_special_tokens':           True,           #Adds [CLS] & [SEP]\n",
        "    'truncation_strategy':          'only_second',  #Only body of article is cut\n",
        "    'return_token_type_ids':        True,           #when using pairs, adds the type indicator\n",
        "    'return_overflowing_tokens':    False,          #allows for overflowing tokens\n",
        "    'return_special_tokens_mask':   True,           #masking tokens\n",
        "    'return_tensors':               'pt'            #return as torch tensor\n",
        "}\n",
        "pad_c = tokenizer.special_tokens_map['pad_token']\n",
        "sep_c = tokenizer.special_tokens_map['sep_token']\n",
        "options['max_length'] += 1\n",
        "\n",
        "def processInput(raw_batch):\n",
        "    id_batch = []\n",
        "    type_batch = []\n",
        "    mask_batch = []\n",
        "    label_batch = []\n",
        "    for sent1,sent2,label in raw_batch:\n",
        "        enc = tokenizer.encode_plus(sent1,sent2+sep_c+pad_c*options['max_length'],**options)\n",
        "\n",
        "        token_id = enc['input_ids'][0][:options['max_length']-1]\n",
        "        token_type = enc['token_type_ids'][0][:options['max_length']-1]\n",
        "        mask = token_id.clone()\n",
        "        mask[mask != 0] = 1\n",
        "\n",
        "        id_batch.append(token_id.to(device))\n",
        "        type_batch.append(token_type.to(device))\n",
        "        mask_batch.append(mask.to(device))\n",
        "        label_batch.append(classlist[label])\n",
        "    return (torch.stack(id_batch),torch.stack(type_batch),torch.stack(mask_batch),torch.tensor(label_batch,device=device))\n",
        "#processInput(df.head(5).apply(lambda row: (row.sentence1,row.sentence2,row.gold_label),axis=1))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TaWwdy4ztfQn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import random\n",
        "seed = 666\n",
        "random.seed(seed)\n",
        "batch_size = 16\n",
        "\n",
        "def train(model,data,epochs=1):\n",
        "    optimizer = AdamW(model.parameters())\n",
        "    head_mask = [0]+ [1] * (model.config.num_attention_heads-1)\n",
        "    model.train()\n",
        "    for i in range(epochs):\n",
        "        print('EPOCH %d'%i)\n",
        "        loader = data.sample(frac=1,random_state=seed).reset_index(drop=True).groupby(np.arange(len(data))//batch_size)\n",
        "        tr_loss = 0\n",
        "        print('Total batches: %d'%len(loader))\n",
        "        for g, df in loader:\n",
        "            print('Batch %d'%g)\n",
        "            id_batch,type_batch,mask_batch,label_batch = processInput(df.apply(lambda row: (row.sentence1,row.sentence2,row.gold_label),axis=1))\n",
        "            random.shuffle(head_mask)\n",
        "            headm = torch.tensor(head_mask,device=device)\n",
        "\n",
        "            model.zero_grad()\n",
        "            optimizer.zero_grad()\n",
        "\n",
        "            loss, logits = model(input_ids=id_batch,\n",
        "                                attention_mask=mask_batch,\n",
        "                                token_type_ids=type_batch,\n",
        "                                labels=label_batch,\n",
        "                                head_mask=headm)[:2]\n",
        "            \n",
        "            loss.backward()\n",
        "            tr_loss += loss.item()\n",
        "            optimizer.step()\n",
        "            print('Loss %f'%(tr_loss/(g+1)))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CoyXV71PymVF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "87084992-3476-470c-fb4d-a3a420b16dda"
      },
      "source": [
        "train(model,df)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "EPOCH 0\n",
            "Total batches: 34385\n",
            "Batch 0\n",
            "Loss 1.043603\n",
            "Batch 1\n",
            "Loss 1.127185\n",
            "Batch 2\n",
            "Loss 1.156964\n",
            "Batch 3\n",
            "Loss 1.145500\n",
            "Batch 4\n",
            "Loss 1.225963\n",
            "Batch 5\n",
            "Loss 1.219158\n",
            "Batch 6\n",
            "Loss 1.231108\n",
            "Batch 7\n",
            "Loss 1.241617\n",
            "Batch 8\n",
            "Loss 1.243058\n",
            "Batch 9\n",
            "Loss 1.298170\n",
            "Batch 10\n",
            "Loss 1.296156\n",
            "Batch 11\n",
            "Loss 1.278732\n",
            "Batch 12\n",
            "Loss 1.268191\n",
            "Batch 13\n",
            "Loss 1.255250\n",
            "Batch 14\n",
            "Loss 1.238127\n",
            "Batch 15\n",
            "Loss 1.248640\n",
            "Batch 16\n",
            "Loss 1.258103\n",
            "Batch 17\n",
            "Loss 1.260792\n",
            "Batch 18\n",
            "Loss 1.249558\n",
            "Batch 19\n",
            "Loss 1.244225\n",
            "Batch 20\n",
            "Loss 1.233132\n",
            "Batch 21\n",
            "Loss 1.224359\n",
            "Batch 22\n",
            "Loss 1.230350\n",
            "Batch 23\n",
            "Loss 1.230384\n",
            "Batch 24\n",
            "Loss 1.227822\n",
            "Batch 25\n",
            "Loss 1.223148\n",
            "Batch 26\n",
            "Loss 1.219139\n",
            "Batch 27\n",
            "Loss 1.216233\n",
            "Batch 28\n",
            "Loss 1.213799\n",
            "Batch 29\n",
            "Loss 1.211235\n",
            "Batch 30\n",
            "Loss 1.213236\n",
            "Batch 31\n",
            "Loss 1.217097\n",
            "Batch 32\n",
            "Loss 1.219258\n",
            "Batch 33\n",
            "Loss 1.214717\n",
            "Batch 34\n",
            "Loss 1.213100\n",
            "Batch 35\n",
            "Loss 1.213830\n",
            "Batch 36\n",
            "Loss 1.212602\n",
            "Batch 37\n",
            "Loss 1.209842\n",
            "Batch 38\n",
            "Loss 1.207621\n",
            "Batch 39\n",
            "Loss 1.204586\n",
            "Batch 40\n",
            "Loss 1.200721\n",
            "Batch 41\n",
            "Loss 1.198711\n",
            "Batch 42\n",
            "Loss 1.195518\n",
            "Batch 43\n",
            "Loss 1.194539\n",
            "Batch 44\n",
            "Loss 1.193213\n",
            "Batch 45\n",
            "Loss 1.189737\n",
            "Batch 46\n",
            "Loss 1.188226\n",
            "Batch 47\n",
            "Loss 1.190398\n",
            "Batch 48\n",
            "Loss 1.187467\n",
            "Batch 49\n",
            "Loss 1.187459\n",
            "Batch 50\n",
            "Loss 1.185528\n",
            "Batch 51\n",
            "Loss 1.185619\n",
            "Batch 52\n",
            "Loss 1.183044\n",
            "Batch 53\n",
            "Loss 1.185984\n",
            "Batch 54\n",
            "Loss 1.183013\n",
            "Batch 55\n",
            "Loss 1.183357\n",
            "Batch 56\n",
            "Loss 1.182871\n",
            "Batch 57\n",
            "Loss 1.181245\n",
            "Batch 58\n",
            "Loss 1.179861\n",
            "Batch 59\n",
            "Loss 1.179353\n",
            "Batch 60\n",
            "Loss 1.177859\n",
            "Batch 61\n",
            "Loss 1.176796\n",
            "Batch 62\n",
            "Loss 1.175615\n",
            "Batch 63\n",
            "Loss 1.176347\n",
            "Batch 64\n",
            "Loss 1.180122\n",
            "Batch 65\n",
            "Loss 1.180603\n",
            "Batch 66\n",
            "Loss 1.180297\n",
            "Batch 67\n",
            "Loss 1.179679\n",
            "Batch 68\n",
            "Loss 1.178813\n",
            "Batch 69\n",
            "Loss 1.178502\n",
            "Batch 70\n",
            "Loss 1.178531\n",
            "Batch 71\n",
            "Loss 1.179295\n",
            "Batch 72\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cr4aAQzGyqrK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bTCBEKPT1bs-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}