Translation Models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Translation Models for Question Retrieval\n",
    "\n",
    "### Loading dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import logging\n",
    "from gensim import corpora\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "#logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO) # logging"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading Stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path = '/roaming/fkunnema/goeievraag/exp_similarity/stopwords.txt'\n",
    "with open(path) as f:\n",
    "    stopwords = [word.lower().strip() for word in f.read().split()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading Goeievraag corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of train questions:  497054\n",
      "Number of test questions:  100\n"
     ]
    }
   ],
   "source": [
    "goeievraag_path = '/roaming/fkunnema/goeievraag'  # goeievraag path\n",
    "# loading tokenized and 'stopworded' questions, and lowercase them.\n",
    "train_question_path = os.path.join(goeievraag_path, 'exp_similarity', 'train_questions.tok.txt')\n",
    "with open(train_question_path) as f:\n",
    "    doc = f.read()\n",
    "\n",
    "original_train_questions, train_questions = [], []\n",
    "for question in doc.split('\\n')[:-1]:\n",
    "    original_question = [word for word in question.lower().split()]\n",
    "    question = [word for word in original_question if word not in stopwords]\n",
    "    if len(question) > 0:\n",
    "        original_train_questions.append(original_question)\n",
    "        train_questions.append(question)\n",
    "print('Number of train questions: ', str(len(train_questions)))\n",
    "\n",
    "path = os.path.join(goeievraag_path, 'exp_similarity', 'seed_questions.tok.txt')\n",
    "with open(path) as f:\n",
    "    doc = f.read()\n",
    "\n",
    "original_test_questions, test_questions = [], []\n",
    "for question in doc.split('\\n')[:-1]:\n",
    "    original_question = [word for word in question.lower().split()]\n",
    "    question = [word for word in original_question if word not in stopwords]\n",
    "    if len(question) > 0:\n",
    "        original_test_questions.append(original_question)\n",
    "        test_questions.append(question)\n",
    "print('Number of test questions: ', str(len(test_questions)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Computing and saving dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vocabulary = corpora.Dictionary(train_questions)\n",
    "#vocabulary.save('goeievraag.dict')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Background language model \n",
    "P(w | C)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# question unigrams P(w | C)\n",
    "tokens = []\n",
    "for question in train_questions:\n",
    "    for token in question:\n",
    "        tokens.append(token)\n",
    "\n",
    "vocablen = len(vocabulary)\n",
    "Q_len = float(len(tokens))\n",
    "aux_w_Q = vocabulary.doc2bow(tokens)\n",
    "aux_w_Q = dict([(vocabulary[w[0]], (w[1]+1.0)/(Q_len+vocablen)) for w in aux_w_Q])\n",
    "\n",
    "w_Q = {}\n",
    "for w in aux_w_Q:\n",
    "    if w[0] not in w_Q:\n",
    "        w_Q[w[0]] = {}\n",
    "    w_Q[w[0]][w] = aux_w_Q[w]\n",
    "\n",
    "# with open('prob_w_Q.txt', 'w') as f:\n",
    "#     for w, prob in w_Q:\n",
    "#         f.write(' '.join([w, str(prob), '\\n']))     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.00014471164341808902"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w_Q['m']['muis']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lexical probabilities\n",
    "P(w | t) with indexing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path = '/home/tcastrof/DiscoSumo/translation/model/lex.f2e'\n",
    "with open(path) as f:\n",
    "    doc = list(map(lambda x: x.split(), f.read().split('\\n')))\n",
    "    \n",
    "t2w = {}\n",
    "for row in doc[:-1]:\n",
    "    t = row[0]\n",
    "    if t[0] not in t2w:\n",
    "        t2w[t[0]] = {}\n",
    "    if t not in t2w[t[0]]:\n",
    "        t2w[t[0]][t] = {}\n",
    "    \n",
    "    w = row[1]\n",
    "    if w[0] not in t2w[t[0]][t]:\n",
    "        t2w[t[0]][t][w[0]] = {}\n",
    "    \n",
    "    prob = float(row[2])\n",
    "    t2w[t[0]][t][w[0]][w] = prob"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Base():\n",
    "    def __init__(self, training, prob_w_C, vocablen, alpha):\n",
    "        self.training = training\n",
    "        self.prob_w_C = prob_w_C\n",
    "        self.vocablen = vocablen\n",
    "        self.alpha = alpha\n",
    "\n",
    "    def score(self, query, question, w_Cs=[]):\n",
    "        raise NotImplementedError(\"Please Implement this method\")\n",
    "\n",
    "    def rank(self, query, n=10):\n",
    "        ranking = []\n",
    "\n",
    "        w_Cs = []\n",
    "        for w in query:\n",
    "            w = w.lower()\n",
    "            try:\n",
    "                w_C = self.prob_w_C[w[0]][w]\n",
    "            except:\n",
    "                w_C = 1.0 / self.vocablen\n",
    "            w_Cs.append(w_C)\n",
    "\n",
    "        for i, question in enumerate(self.training):\n",
    "            if i % 1000 == 0:\n",
    "                print('Question: ', str(i), ' '.join(question), sep=\"\\t\")\n",
    "            prob = self.score(query, question, w_Cs)\n",
    "            ranking.append((i, prob))\n",
    "\n",
    "        ranking = sorted(ranking, key=lambda x: x[1], reverse=True)\n",
    "        return ranking[:n]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Language Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class LM(Base):\n",
    "    def __init__(self, training, prob_w_C, vocablen, alpha):\n",
    "        Base.__init__(self, training, prob_w_C, vocablen, alpha)\n",
    "\n",
    "    def score(self, query, question, w_Cs=[]):\n",
    "        Q = pd.Series(question)\n",
    "        Q_count = Q.count()\n",
    "        prob = 0.0\n",
    "        for i, w in enumerate(query):\n",
    "            w = w.lower()\n",
    "            try:\n",
    "                w_Q = float(Q[Q == w].count()) / Q_count\n",
    "            except:\n",
    "                w_Q = 0.0\n",
    "\n",
    "            if len(w_Cs) == 0:\n",
    "                try:\n",
    "                    w_C = self.prob_w_C[w[0]][w]\n",
    "                except:\n",
    "                    w_C = 1.0 / self.vocablen\n",
    "            else:\n",
    "                w_C = w_Cs[i]\n",
    "\n",
    "            prob += np.log(((1-self.alpha) * w_Q) + (self.alpha * w_C))\n",
    "        return prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: \t0\ti like bananas\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[(2, -16.3987280303173), (0, -28.83220048974483), (1, -28.83220048974483)]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training = ['i like bananas', 'i like apples', 'i hate aubergine']\n",
    "training = map(lambda s: s.split(), training)\n",
    "\n",
    "prob_w_C = {\n",
    "    'i': 0.1,\n",
    "    'like': 0.2,\n",
    "    'bananas': 0.15,\n",
    "    'apples': 0.15,\n",
    "    'hate': 0.2,\n",
    "    'aubergine': 0.1,\n",
    "    'pineapple': 0.1\n",
    "}\n",
    "\n",
    "alpha = 0.2\n",
    "\n",
    "lm = LM(training, prob_w_C, len(vocabulary), alpha)\n",
    "lm.rank('i hate pineapple'.split())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Translation Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class TRM(Base):\n",
    "    def __init__(self, training, prob_w_C, prob_w_t, vocablen, alpha):\n",
    "        Base.__init__(self, training, prob_w_C, vocablen, alpha) \n",
    "        self.prob_w_t = prob_w_t\n",
    "        \n",
    "    def score(self, query, question, w_Cs=[]):\n",
    "        Q = pd.Series(question)\n",
    "        Q_count = Q.count()\n",
    "        \n",
    "        t_Qs = []\n",
    "        for t in question:\n",
    "            t = t.lower()\n",
    "            t_Q = float(Q[Q == t].count()) / Q_count\n",
    "            t_Qs.append(t_Q)\n",
    "        \n",
    "        prob = 0.0\n",
    "        for i, w in enumerate(query):\n",
    "            w = w.lower()\n",
    "            if len(w_Cs) == 0:\n",
    "                try:\n",
    "                    w_C = self.prob_w_C[w[0]][w]\n",
    "                except:\n",
    "                    w_C = 1.0 / self.vocablen\n",
    "            else:\n",
    "                w_C = w_Cs[i]\n",
    "            \n",
    "            w_Q = 0.0\n",
    "            for j, t in enumerate(question):\n",
    "                t = t.lower()\n",
    "                try:\n",
    "                    w_t = prob_w_t[t[0]][t][w[0]][w]\n",
    "                except:\n",
    "                    w_t = 0.0\n",
    "                \n",
    "                t_Q = t_Qs[j]\n",
    "                w_Q += (w_t * t_Q)\n",
    "            prob += np.log(((1-self.alpha) * w_Q) + (self.alpha * w_C))\n",
    "        return prob"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Translation-based Language Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class TRLM(Base):\n",
    "    def __init__(self, training, prob_w_C, prob_w_t, vocablen, alpha, sigma):\n",
    "        Base.__init__(self, training, prob_w_C, vocablen, alpha) \n",
    "        self.prob_w_t = prob_w_t\n",
    "        self.sigma = sigma\n",
    "    \n",
    "    def score_all(self, query, question, w_Cs=[]):\n",
    "        Q = pd.Series(question)\n",
    "        Q_count = Q.count()\n",
    "        \n",
    "        t_Qs = []\n",
    "        for t in question:\n",
    "            t = t.lower()\n",
    "            t_Q = float(Q[Q == t].count()) / Q_count\n",
    "            t_Qs.append(t_Q)\n",
    "        \n",
    "        lmprob, trmprob, trlmprob = 0.0, 0.0, 0.0\n",
    "        for i, w in enumerate(query):\n",
    "            w = w.lower()\n",
    "            if len(w_Cs) == 0:\n",
    "                try:\n",
    "                    w_C = self.prob_w_C[w[0]][w]\n",
    "                except:\n",
    "                    w_C = 1.0 / self.vocablen\n",
    "            else:\n",
    "                w_C = w_Cs[i]\n",
    "            \n",
    "            ml_w_Q = float(Q[Q == w].count()) / Q_count\n",
    "            mx_w_Q = 0.0\n",
    "            for j, t in enumerate(question):\n",
    "                t = t.lower()\n",
    "                try:\n",
    "                    w_t = self.prob_w_t[t[0]][t][w[0]][w]\n",
    "                except:\n",
    "                    w_t = 0.0\n",
    "                print(t, w, str(w_t))\n",
    "                \n",
    "                t_Q = t_Qs[j]\n",
    "                mx_w_Q += (w_t * t_Q)\n",
    "            w_Q = (self.sigma * mx_w_Q) + ((1-self.sigma) * ml_w_Q)\n",
    "            lmprob += np.log(((1-self.alpha) * ml_w_Q) + (self.alpha * w_C))\n",
    "            trmprob += np.log(((1-self.alpha) * mx_w_Q) + (self.alpha * w_C))\n",
    "            trlmprob += np.log(((1-self.alpha) * w_Q) + (self.alpha * w_C))\n",
    "        return lmprob, trmprob, trlmprob\n",
    "    \n",
    "    def rank_all(self, query, n=10):\n",
    "        lmrank, trmrank, trlmrank = [], [], []\n",
    "        score_times = []\n",
    "\n",
    "        w_Cs = []\n",
    "        for w in query:\n",
    "            w = w.lower()\n",
    "            try:\n",
    "                w_C = self.prob_w_C[w[0]][w]\n",
    "            except:\n",
    "                w_C = 1.0 / self.vocablen\n",
    "            w_Cs.append(w_C)\n",
    "\n",
    "        for i, question in enumerate(self.training):\n",
    "            lmprob, trmprob, trlmprob, score_time  = self.score_all(query, question, w_Cs)\n",
    "            score_times.append(score_time)\n",
    "            lmrank.append((i, lmprob))\n",
    "            trmrank.append((i, trmprob))\n",
    "            trlmrank.append((i, trlmprob))\n",
    "            if i % 1000 == 0:\n",
    "                print('Train question: ', str(i), 'Score time: ', str(np.mean(score_times)), 'Total time: ', str(sum(score_times)), sep=\"\\t\")\n",
    "\n",
    "        lmrank = sorted(lmrank, key=lambda x: x[1], reverse=True)\n",
    "        trmrank = sorted(trmrank, key=lambda x: x[1], reverse=True)\n",
    "        trlmrank = sorted(trlmrank, key=lambda x: x[1], reverse=True)\n",
    "        return lmrank[:n], trmrank[:n], trlmrank[:n]\n",
    "    \n",
    "    def score(self, query, question, w_Cs=[]):\n",
    "        Q = pd.Series(question)\n",
    "        Q_count = Q.count()\n",
    "        \n",
    "        t_Qs = []\n",
    "        for t in question:\n",
    "            t = t.lower()\n",
    "            t_Q = float(Q[Q == t].count()) / Q_count\n",
    "            t_Qs.append(t_Q)\n",
    "        \n",
    "        prob = 0.0\n",
    "        for i, w in enumerate(query):\n",
    "            w = w.lower()\n",
    "            if len(w_Cs) == 0:\n",
    "                try:\n",
    "                    w_C = self.prob_w_C[w[0]][w]\n",
    "                except:\n",
    "                    w_C = 0.0\n",
    "            else:\n",
    "                w_C = w_Cs[i]\n",
    "            \n",
    "            ml_w_Q = float(Q[Q == w].count()) / Q_count\n",
    "            mx_w_Q = 0.0\n",
    "            for j, t in enumerate(question):\n",
    "                t = t.lower()\n",
    "                try:\n",
    "                    w_t = self.prob_w_t[t[0]][t][w[0]][w]\n",
    "                except:\n",
    "                    w_t = 0.0\n",
    "                \n",
    "                t_Q = t_Qs[j]\n",
    "                mx_w_Q += (w_t * t_Q)\n",
    "            w_Q = (self.sigma * mx_w_Q) + ((1-self.sigma) * ml_w_Q)\n",
    "            prob += np.log(((1-self.alpha) * w_Q) + (self.alpha * w_C))\n",
    "        return prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "wat wat 0.0308996\n",
      "zijn wat 0.006492\n",
      "voortekenen wat 4.2e-06\n",
      "van wat 0.011798\n",
      "een wat 0.1310603\n",
      "psychose wat 0.0\n",
      "wat is 0.0016978\n",
      "zijn is 0.0163889\n",
      "voortekenen is 0.0\n",
      "van is 0.0102445\n",
      "een is 0.0308229\n",
      "psychose is 0.0\n",
      "wat een 0.0004342\n",
      "zijn een 0.0001112\n",
      "voortekenen een 0.0\n",
      "van een 0.0372428\n",
      "een een 0.2483838\n",
      "psychose een 1.47e-05\n",
      "wat toxische 0.0\n",
      "zijn toxische 0.0\n",
      "voortekenen toxische 0.0\n",
      "van toxische 0.0\n",
      "een toxische 0.0\n",
      "psychose toxische 0.0\n",
      "wat psychose 0.035533\n",
      "zijn psychose 0.0\n",
      "voortekenen psychose 0.0\n",
      "van psychose 0.0\n",
      "een psychose 0.0\n",
      "psychose psychose 0.1167513\n",
      "(-34.58531746865747, -31.904206279546965, -29.077728157957758)\n"
     ]
    }
   ],
   "source": [
    "tq1 = 'wat zijn voortekenen van een psychose'.split()\n",
    "tq2 = 'wat is een begripsvraag wat is een toepassingsvraag wat is een kennisvraag'.split()\n",
    "trm = TRLM([tq1], w_Q, t2w, len(vocabulary), 0.5, 0.3)\n",
    "\n",
    "query = 'wat is een toxische psychose'.split()\n",
    "print(trm.score_all(query, tq1))\n",
    "# print(trm.score(query, tq2))\n",
    "# print(trm.score(tq2, tq1))\n",
    "# print(trm.score(tq2, tq2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Script"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}