Non_Deep_Baseline.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Non-Deep-Baseline.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ymann/Machine-Comprehension-Cloze-QA/blob/master/Non_Deep_Baseline.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "Eb_DrywYHsea",
        "colab_type": "code",
        "outputId": "98fba2e5-c8df-4ac3-889b-c0ac3f70bc9b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 303
        }
      },
      "cell_type": "code",
      "source": [
        "! wget http://seas.upenn.edu/~branlin/cnn.tgz \n",
        "! tar -xzf cnn.tgz"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2019-04-23 01:01:39--  http://seas.upenn.edu/~branlin/cnn.tgz\n",
            "Resolving seas.upenn.edu (seas.upenn.edu)... 158.130.68.91\n",
            "Connecting to seas.upenn.edu (seas.upenn.edu)|158.130.68.91|:80... connected.\n",
            "HTTP request sent, awaiting response... 301 Moved Permanently\n",
            "Location: https://www.seas.upenn.edu/~branlin/cnn.tgz [following]\n",
            "--2019-04-23 01:01:39--  https://www.seas.upenn.edu/~branlin/cnn.tgz\n",
            "Resolving www.seas.upenn.edu (www.seas.upenn.edu)... 158.130.68.91, 2607:f470:8:64:5ea5::9\n",
            "Connecting to www.seas.upenn.edu (www.seas.upenn.edu)|158.130.68.91|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 217447312 (207M) [application/x-gzip]\n",
            "Saving to: ‘cnn.tgz’\n",
            "\n",
            "cnn.tgz             100%[===================>] 207.37M  18.5MB/s    in 12s     \n",
            "\n",
            "2019-04-23 01:01:51 (17.7 MB/s) - ‘cnn.tgz’ saved [217447312/217447312]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "MEKp_8FConPP",
        "colab_type": "code",
        "outputId": "9e0ef8bc-d386-4f9e-f6c1-f2372fcb4c54",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 124
        }
      },
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/cis700/hw1-release.git\n",
        "!mv hw1-release/helper.py .\n",
        "!rm -rf hw1-release/\n",
        "!rm -rf hw1/"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'hw1-release'...\n",
            "remote: Enumerating objects: 26, done.\u001b[K\n",
            "remote: Counting objects:   3% (1/26)   \u001b[K\rremote: Counting objects:   7% (2/26)   \u001b[K\rremote: Counting objects:  11% (3/26)   \u001b[K\rremote: Counting objects:  15% (4/26)   \u001b[K\rremote: Counting objects:  19% (5/26)   \u001b[K\rremote: Counting objects:  23% (6/26)   \u001b[K\rremote: Counting objects:  26% (7/26)   \u001b[K\rremote: Counting objects:  30% (8/26)   \u001b[K\rremote: Counting objects:  34% (9/26)   \u001b[K\rremote: Counting objects:  38% (10/26)   \u001b[K\rremote: Counting objects:  42% (11/26)   \u001b[K\rremote: Counting objects:  46% (12/26)   \u001b[K\rremote: Counting objects:  50% (13/26)   \u001b[K\rremote: Counting objects:  53% (14/26)   \u001b[K\rremote: Counting objects:  57% (15/26)   \u001b[K\rremote: Counting objects:  61% (16/26)   \u001b[K\rremote: Counting objects:  65% (17/26)   \u001b[K\rremote: Counting objects:  69% (18/26)   \u001b[K\rremote: Counting objects:  73% (19/26)   \u001b[K\rremote: Counting objects:  76% (20/26)   \u001b[K\rremote: Counting objects:  80% (21/26)   \u001b[K\rremote: Counting objects:  84% (22/26)   \u001b[K\rremote: Counting objects:  88% (23/26)   \u001b[K\rremote: Counting objects:  92% (24/26)   \u001b[K\rremote: Counting objects:  96% (25/26)   \u001b[K\rremote: Counting objects: 100% (26/26)   \u001b[K\rremote: Counting objects: 100% (26/26), done.\u001b[K\n",
            "remote: Compressing objects:   5% (1/17)   \u001b[K\rremote: Compressing objects:  11% (2/17)   \u001b[K\rremote: Compressing objects:  17% (3/17)   \u001b[K\rremote: Compressing objects:  23% (4/17)   \u001b[K\rremote: Compressing objects:  29% (5/17)   \u001b[K\rremote: Compressing objects:  35% (6/17)   \u001b[K\rremote: Compressing objects:  41% (7/17)   \u001b[K\rremote: Compressing objects:  47% (8/17)   \u001b[K\rremote: Compressing objects:  52% (9/17)   \u001b[K\rremote: Compressing objects:  58% (10/17)   \u001b[K\rremote: Compressing objects:  64% (11/17)   \u001b[K\rremote: Compressing objects:  70% (12/17)   \u001b[K\rremote: Compressing objects:  76% (13/17)   \u001b[K\rremote: Compressing objects:  82% (14/17)   \u001b[K\rremote: Compressing objects:  88% (15/17)   \u001b[K\rremote: Compressing objects:  94% (16/17)   \u001b[K\rremote: Compressing objects: 100% (17/17)   \u001b[K\rremote: Compressing objects: 100% (17/17), done.\u001b[K\n",
            "remote: Total 26 (delta 6), reused 23 (delta 6), pack-reused 0\u001b[K\n",
            "Unpacking objects:   3% (1/26)   \rUnpacking objects:   7% (2/26)   \rUnpacking objects:  11% (3/26)   \rUnpacking objects:  15% (4/26)   \rUnpacking objects:  19% (5/26)   \rUnpacking objects:  23% (6/26)   \rUnpacking objects:  26% (7/26)   \rUnpacking objects:  30% (8/26)   \rUnpacking objects:  34% (9/26)   \rUnpacking objects:  38% (10/26)   \rUnpacking objects:  42% (11/26)   \rUnpacking objects:  46% (12/26)   \rUnpacking objects:  50% (13/26)   \rUnpacking objects:  53% (14/26)   \rUnpacking objects:  57% (15/26)   \rUnpacking objects:  61% (16/26)   \rUnpacking objects:  65% (17/26)   \rUnpacking objects:  69% (18/26)   \rUnpacking objects:  73% (19/26)   \rUnpacking objects:  76% (20/26)   \rUnpacking objects:  80% (21/26)   \rUnpacking objects:  84% (22/26)   \rUnpacking objects:  88% (23/26)   \rUnpacking objects:  92% (24/26)   \rUnpacking objects:  96% (25/26)   \rUnpacking objects: 100% (26/26)   \rUnpacking objects: 100% (26/26), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "vlW8S1bMoaRP",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "LOG_DIR = './logs'\n",
        "get_ipython().system_raw(\n",
        "    'tensorboard --logdir {} --host 0.0.0.0 --port 6006 &'\n",
        "    .format(LOG_DIR)\n",
        ")\n",
        "\n",
        "!if [ -f ngrok ] ; then echo \"Ngrok already installed\" ; else wget https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip > /dev/null 2>&1 && unzip ngrok-stable-linux-amd64.zip > /dev/null 2>&1 ; fi\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "h68zP3nhohEM",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "get_ipython().system_raw('./ngrok http 6006 &')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "xVdX4qr8g9IW",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "LS3Mgx2Hojq4",
        "colab_type": "code",
        "outputId": "f28fd4b4-23f1-487a-9976-50e572f119fc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "! curl -s http://localhost:4040/api/tunnels | python3 -c \\\n",
        "    \"import sys, json; print('Tensorboard Link: ' +str(json.load(sys.stdin)['tunnels'][0]['public_url']))\""
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tensorboard Link: https://2ab70209.ngrok.io\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "Azhipc8KI7Nk",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import torch.nn as nn\n",
        "import torch.utils.data\n",
        "import os\n",
        "from collections import Counter\n",
        "from nltk import ngrams\n",
        "\n",
        "class QuestionDataset(torch.utils.data.Dataset):\n",
        "  \n",
        "  def __init__(self, root_dir):\n",
        "    self.root_dir = root_dir\n",
        "    self.files = os.listdir(os.path.join(os.getcwd(), root_dir))\n",
        "    \n",
        "  def __len__(self):\n",
        "    return len(self.files)\n",
        "  \n",
        "  def __getitem__(self, idx):\n",
        "    with open(os.path.join(self.root_dir, self.files[idx]), 'r') as f:\n",
        "      s = f.read()\n",
        "    \n",
        "    lines = s.split('\\n')\n",
        "    doc = lines[2]\n",
        "    tokens = doc.split()\n",
        "    query = lines[4]\n",
        "    query_tokens = query.split()\n",
        "    correct_entity = lines[6]\n",
        "    entities = [tuple(i.split(\":\")) for i in lines[8:]]\n",
        "    return (tokens, query, correct_entity, entities)\n",
        "  \n",
        "class LogisticRegression(nn.Module):\n",
        "  \n",
        "  def __init__(self, num_classes=8):\n",
        "    super(LogisticRegression, self).__init__()\n",
        "    \n",
        "    self.fc = nn.Sequential(\n",
        "        nn.Linear(num_classes, 1),\n",
        "        nn.Sigmoid()\n",
        "    )\n",
        "    \n",
        "  def forward(self, x):\n",
        "    return self.fc(x)\n",
        "  \n",
        "# We extract an entity e from the document d and query q based on the following features:\n",
        "# 1. how many times e appears in d\n",
        "# 2. where e is first mentioned in d\n",
        "# 3. proximity match: whether [e, __, w] appears in d, where w is the word succeeding @placeholder and __ is 0 to 2 words. Feature will be the number of spaces\n",
        "# 4. proximity: whether [w, __, e] appears in d, where w is word preceding @placeholder. Feature will be number of spaces\n",
        "\n",
        "def extract_features(tokens, query, correct_entity, entities):\n",
        "  '''\n",
        "  entity should be of the form (@entity<n>, word) as a tuple\n",
        "  '''\n",
        "  \n",
        "  ctr = Counter(tokens)\n",
        "  placeholder_index = query.index('@placeholder')\n",
        "  word_before = query[placeholder_index - 1] if placeholder_index > 0 else None\n",
        "  word_after = query[placeholder_index + 1] if placeholder_index < len(query) else None\n",
        "  \n",
        "  # Generate n grams for the passage, where n ranges from 2 to 4.\n",
        "  bigrams = [(i, 1) for i in ngrams(tokens, 2)]\n",
        "  trigrams = [(i, 2) for i in ngrams(tokens, 3)]\n",
        "  quadgrams = [(i, 3) for i in ngrams(tokens, 4)]\n",
        "  grams = bigrams + trigrams + quadgrams\n",
        "  \n",
        "  features = []\n",
        "  for e in entities:\n",
        "    entity_count = ctr[e[0]]\n",
        "    try:\n",
        "      first_mention = tokens.index(e[0])\n",
        "    except ValueError:\n",
        "      first_mention = -1\n",
        "    prox_before, prox_after = 0, 0 # represents if no proximity match\n",
        "    for gram in grams:\n",
        "      if gram[0][0] == word_before and gram[0][-1] == e[0]:\n",
        "        prox_before = gram[1]\n",
        "        break\n",
        "    for gram in grams:\n",
        "      if gram[0][0] == e[0] and gram[0][-1] == word_after:\n",
        "        prox_after = gram[1]\n",
        "        break\n",
        "    features.append((torch.Tensor([entity_count, first_mention, prox_before, prox_after]), torch.Tensor([1 if correct_entity == e[0] else 0])))\n",
        "  \n",
        "  return features"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "UZoBvzoJzmUM",
        "colab_type": "code",
        "outputId": "63547637-e1ae-4af2-cbe5-d54b9fbb7e26",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 552
        }
      },
      "cell_type": "code",
      "source": [
        "from torch.utils.data import DataLoader\n",
        "import torch.nn as nn\n",
        "\n",
        "class AuxiliaryQuestionsDataset(torch.utils.data.Dataset):\n",
        "  \n",
        "  def __init__(self, subset='train'):\n",
        "    if subset == 'train':\n",
        "      dataset = QuestionDataset(\"cnn/questions/training\")\n",
        "    elif subset == 'test':  \n",
        "      dataset = QuestionDataset(\"cnn/questions/test\")\n",
        "    elif subset == 'validation':\n",
        "      dataset = QuestionDataset(\"cnn/questions/validation\")\n",
        "    else:\n",
        "      raise ValueError(\"Invalid arguments passed; expected train or test.\")\n",
        "      \n",
        "    self.data = []\n",
        "    for i, (tokens, query, correct_entity, entities) in enumerate(dataset):\n",
        "      if i % 10000 == 0:\n",
        "        print(i)\n",
        "      self.data.extend(extract_features(tokens, query, correct_entity, entities))\n",
        "  \n",
        "  def __len__(self):\n",
        "    return len(self.data)\n",
        "  \n",
        "  def __getitem__(self, idx):\n",
        "    return self.data[idx]\n",
        "  \n",
        "  \n",
        "train_data = AuxiliaryQuestionsDataset(subset='train')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0\n",
            "10000\n",
            "20000\n",
            "30000\n",
            "40000\n",
            "50000\n",
            "60000\n",
            "70000\n",
            "80000\n",
            "90000\n",
            "100000\n",
            "110000\n",
            "120000\n",
            "130000\n",
            "140000\n",
            "150000\n",
            "160000\n",
            "170000\n",
            "180000\n",
            "190000\n",
            "200000\n",
            "210000\n",
            "220000\n",
            "230000\n",
            "240000\n",
            "250000\n",
            "260000\n",
            "270000\n",
            "280000\n",
            "290000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "26SK3y-QfqBS",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from torch.utils.data import DataLoader\n",
        "import torch.nn as nn\n",
        "import torch\n",
        "from torch import Tensor as tensor\n",
        "\n",
        "class AuxiliaryQuestionsDataset(torch.utils.data.Dataset):\n",
        "  \n",
        "  def __init__(self, subset='train'):\n",
        "#     if subset == 'train':\n",
        "#       dataset = QuestionDataset(\"cnn/questions/training\")\n",
        "#     elif subset == 'test':  \n",
        "#       dataset = QuestionDataset(\"cnn/questions/test\")\n",
        "#     elif subset == 'validation':\n",
        "#       dataset = QuestionDataset(\"cnn/questions/validation\")\n",
        "#     else:\n",
        "#       raise ValueError(\"Invalid arguments passed; expected train or test.\")\n",
        "      \n",
        "    with open('logregfeatures.txt', 'r') as f:\n",
        "      self.lines = f.readlines()\n",
        "  def __len__(self):\n",
        "    return len(self.lines)\n",
        "  \n",
        "  def __getitem__(self, idx):\n",
        "    tup = eval(self.lines[idx])\n",
        "    return tup\n",
        "  \n",
        "  \n",
        "train_data = AuxiliaryQuestionsDataset(subset='train')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "BqsLDdtfVFch",
        "colab_type": "code",
        "outputId": "dcd7a129-d4d2-469a-ab17-85d6d42475bb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "train_data"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<__main__.AuxiliaryQuestionsDataset at 0x7f783035eb00>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "metadata": {
        "id": "WE9Fd_O1pC1Z",
        "colab_type": "code",
        "outputId": "6cbc9aee-d130-4454-a658-1c5a1d1a379e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "# http://pytorch.org/\n",
        "# from os.path import exists\n",
        "# from wheel.pep425tags import get_abbr_impl, get_impl_ver, get_abi_tag\n",
        "# platform = '{}{}-{}'.format(get_abbr_impl(), get_impl_ver(), get_abi_tag())\n",
        "# cuda_output = !ldconfig -p|grep cudart.so|sed -e 's/.*\\.cd\\([0-9]*\\)\\.\\([0-9]*\\)$/cu\\1\\2/'\n",
        "# accelerator = cuda_output[0] if exists('/dev/nvidia0') else 'cpu'\n",
        "# !pip3 install https://download.pytorch.org/whl/cu100/torch-1.0.1-cp36-cp36m-linux_x86_64.whl\n",
        "# !pip3 install torchvision\n",
        "  \n",
        "import torch\n",
        "device =  torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "print(device)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cuda:0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "-w0N6UJ1kACT",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Code referenced from https://gist.github.com/gyglim/1f8dfb1b5c82627ae3efcfbbadb9f514\n",
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "import scipy.misc \n",
        "try:\n",
        "    from StringIO import StringIO  # Python 2.7\n",
        "except ImportError:\n",
        "    from io import BytesIO         # Python 3.x\n",
        "\n",
        "\n",
        "class Logger(object):\n",
        "    \n",
        "    def __init__(self, log_dir):\n",
        "        \"\"\"Create a summary writer logging to log_dir.\"\"\"\n",
        "        self.writer = tf.summary.FileWriter(log_dir)\n",
        "\n",
        "    def scalar_summary(self, tag, value, step):\n",
        "        \"\"\"Log a scalar variable.\"\"\"\n",
        "        summary = tf.Summary(value=[tf.Summary.Value(tag=tag, simple_value=value)])\n",
        "        self.writer.add_summary(summary, step)\n",
        "\n",
        "    def image_summary(self, tag, images, step):\n",
        "        \"\"\"Log a list of images.\"\"\"\n",
        "\n",
        "        img_summaries = []\n",
        "        for i, img in enumerate(images):\n",
        "            # Write the image to a string\n",
        "            try:\n",
        "                s = StringIO()\n",
        "            except:\n",
        "                s = BytesIO()\n",
        "            scipy.misc.toimage(img).save(s, format=\"png\")\n",
        "\n",
        "            # Create an Image object\n",
        "            img_sum = tf.Summary.Image(encoded_image_string=s.getvalue(),\n",
        "                                       height=img.shape[0],\n",
        "                                       width=img.shape[1])\n",
        "            # Create a Summary value\n",
        "            img_summaries.append(tf.Summary.Value(tag='%s/%d' % (tag, i), image=img_sum))\n",
        "\n",
        "        # Create and write Summary\n",
        "        summary = tf.Summary(value=img_summaries)\n",
        "        self.writer.add_summary(summary, step)\n",
        "        \n",
        "    def histo_summary(self, tag, values, step, bins=1000):\n",
        "        \"\"\"Log a histogram of the tensor of values.\"\"\"\n",
        "\n",
        "        # Create a histogram using numpy\n",
        "        counts, bin_edges = np.histogram(values, bins=bins)\n",
        "\n",
        "        # Fill the fields of the histogram proto\n",
        "        hist = tf.HistogramProto()\n",
        "        hist.min = float(np.min(values))\n",
        "        hist.max = float(np.max(values))\n",
        "        hist.num = int(np.prod(values.shape))\n",
        "        hist.sum = float(np.sum(values))\n",
        "        hist.sum_squares = float(np.sum(values**2))\n",
        "\n",
        "        # Drop the start of the first bin\n",
        "        bin_edges = bin_edges[1:]\n",
        "\n",
        "        # Add bin edges and counts\n",
        "        for edge in bin_edges:\n",
        "            hist.bucket_limit.append(edge)\n",
        "        for c in counts:\n",
        "            hist.bucket.append(c)\n",
        "\n",
        "        # Create and write Summary\n",
        "        summary = tf.Summary(value=[tf.Summary.Value(tag=tag, histo=hist)])\n",
        "        self.writer.add_summary(summary, step)\n",
        "        self.writer.flush()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ykUdLXfvXbtM",
        "colab_type": "code",
        "outputId": "d9c857a1-9775-4e77-e8b5-bf42c9180f20",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import datetime\n",
        "logger = Logger('./logs/run_' + str(datetime.datetime.now()))\n",
        "\n",
        "dataloader = DataLoader(train_data, batch_size=200, shuffle=True)\n",
        "net = LogisticRegression(num_classes=4).to(device)\n",
        "\n",
        "criterion = nn.BCELoss()\n",
        "optimizer = torch.optim.Adam(net.parameters(), lr=0.0001)\n",
        "\n",
        "num_epochs = 5\n",
        "step = 0\n",
        "\n",
        "for epoch in range(num_epochs):\n",
        "  print(\"Running epoch %d\" % epoch)\n",
        "  data_iter = iter(dataloader)\n",
        "  for batch_features, batch_labels in data_iter:\n",
        "    batch_features = batch_features.to(device)\n",
        "    batch_labels = batch_labels.to(device)\n",
        "    output = net(batch_features)\n",
        "    \n",
        "    loss = criterion(output, batch_labels)\n",
        "    optimizer.zero_grad()\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "    \n",
        "    logger.scalar_summary(\"loss\", loss.item(), step+1)\n",
        "    step += 1\n",
        "    "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Running epoch 0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "zO6eJyAsp1C6",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "other_dataset = QuestionDataset('cnn/questions/validation')\n",
        "\n",
        "total_correct, total = 0, 0\n",
        "with torch.no_grad():\n",
        "  for example in other_dataset:\n",
        "    total += 1\n",
        "    ex = extract_features(*example)\n",
        "    features, label = [torch.stack(list(i)) for i in zip(*ex)]\n",
        "    print(features)\n",
        "    features = features.to(device)\n",
        "    label = label.to(device)\n",
        "    output = net(features)\n",
        "    \n",
        "    _, argmax = torch.max(output, 0)\n",
        "    _, argmax2 = torch.max(label, 0)\n",
        "    \n",
        "    if (argmax == argmax2):\n",
        "      total_correct += 1\n",
        "    \n",
        "    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "DJbnfd81wFN2",
        "colab_type": "code",
        "outputId": "2544bcd3-8a1e-4e5c-bb13-b0b07e1d15ae",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "print (total_correct / total)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.290519877675841\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}