phrase segmentation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     /Users/BakerStreetBakery/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('punkt')\n",
    "import math\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 293,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total number of articles 8000\n"
     ]
    }
   ],
   "source": [
    "text = pd.read_csv('./data/Full-Economic-News-DFE-839861.csv')\n",
    "print 'total number of articles', len(text)\n",
    "articles_to_use = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 326,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['new', 'york', 'yields', 'on', 'most', 'certificates', 'of', 'deposit', 'offered', 'by', 'major', 'banks', 'dropped', 'more', 'than', 'a', 'tenth', 'of', 'a', 'percentage', 'point', 'in', 'the', 'latest', 'week', 'reflecting', 'the', 'overall', 'decline', 'in', 'interest', 'rates', 'on', 'or', 'consumer', 'cds', 'sold', 'directly', 'by', 'banks', 'the', 'average', 'yield', 'on', 'deposits', 'fell', 'to', 'from', 'in', 'the', 'week', 'ended', 'yesterday', 'according', 'to', 'an', 'survey', 'by', 'banxquote', 'money', 'markets', 'a', 'wilmington', 'information', 'service', 'on', 'consumer', 'deposits', 'the', 'average', 'yield', 'sank', 'to', 'from', 'the', 'week', 'before', 'according', 'to', 'banxquote', 'two', 'banks', 'in', 'the', 'banxquote', 'survey', 'citibank', 'in', 'new', 'york', 'and', 'corestates', 'in', 'pennsylvania', 'are', 'paying', 'less', 'than', 'on', 'threemonth', 'cds', 'declines', 'were', 'somewhat', 'smaller', 'on', 'consumer', 'cds', 'which', 'eased', 'to', 'from', 'banxquote', 'said', 'yields', 'on', 'and', 'treasury', 'bills', 'sold', 'at', 'monday', 'auction', 'plummeted', 'more', 'than', 'a', 'fifth', 'of', 'a', 'percentage', 'point', 'from', 'the', 'previous', 'week', 'to', 'and', 'respectively', 'the', 'wall', 'street', 'journal', 'online', 'the', 'morning', 'brief', 'a', 'look', 'at']\n",
      "['NEW YORK -- Yields on most certificates of deposit offered by major banks dropped more than a tenth of a percentage point in the latest week, reflecting the overall decline in short-term interest rates.  On small-denomination, or \"consumer,\" CDs sold directly by banks, the average yield on six-month deposits fell to 5.49% from 5.62% in the week ended yesterday, according to an 18-bank survey by Banxquote Money Markets, a Wilmington, Del., information service.  On three-month \"consumer\" deposits, the average yield sank to 5.29% from 5.42% the week before, according to Banxquote. Two banks in the Banxquote survey, Citibank in New York and CoreStates in Pennsylvania, are paying less than 5% on threemonth small-denomination CDs.  Declines were somewhat smaller on five-year consumer CDs, which eased to 7.37% from 7.45%, Banxquote said.  Yields on three-month and six-month Treasury bills sold at Monday\\'s auction plummeted more than a fifth of a percentage point from the previous week, to 5.46% and 5.63%, respectively.']\n",
      "\n",
      "total number of words: 43155\n"
     ]
    }
   ],
   "source": [
    "articles = [x.replace('</br>', ' ') for x in text['text'][:articles_to_use]]\n",
    "all_articles = []\n",
    "for a in articles:\n",
    "    tokens = nltk.word_tokenize(a)\n",
    "    all_articles.append([word.lower() for word in tokens if word.isalpha()])\n",
    "    \n",
    "total_tokens = nltk.word_tokenize(' '.join(articles))\n",
    "all_words = [word.lower() for word in total_tokens if word.isalpha()]\n",
    "\n",
    "print all_words[:150]\n",
    "print articles[:1]\n",
    "print '\\ntotal number of words:', len(all_words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Raw Frequency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 327,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "min_threshold_tau = 5\n",
    "words_len = len(all_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "f = {}\n",
    "index = {}\n",
    "for i in range(words_len):\n",
    "    word = all_words[i]\n",
    "    if word not in index.keys():\n",
    "        index[word] = [i]\n",
    "    else:\n",
    "        index[word].append(i)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 329,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sample words found: ['four', 'mirage', 'adjustors', 'increase', 'eligible', 'electricity', 'sinking', 'regional', 'bringing', 'scholar']\n",
      "sample frequency: [3073, 3943, 4385, 4544, 4637, 6566, 7259, 9210, 10490, 12659, 14185, 14272, 15537, 17272, 17905, 19344, 19753, 20922, 22247, 23219, 23826, 23871, 26225, 26915, 27180, 29750, 29799, 30016, 30540, 31554, 32068, 32082, 32132, 32959, 34327, 37354, 37420, 38332, 40072, 42306]\n"
     ]
    }
   ],
   "source": [
    "print 'sample words found:', index.keys()[:10]\n",
    "print 'sample frequency:', index[index.keys()[3]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[12647, 35904, 36094, 37721, 43154]"
      ]
     },
     "execution_count": 330,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index_allwords = index.copy()\n",
    "index_allwords['risen']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'rate raising', u'is nestled', u'settlement mci', u'and pray', u'also was', u'foreign currency', u'index measure', u'open new', u'at georgetown', u'yield that']\n",
      "[u'to be cut', u'a year what', u'from the company', u'known as quantitative', u'whether the high', u'after a good', u'federal reserve from', u'months ago raising', u'the nasdaq composition', u'according to banxquote']\n",
      "[u'over the past three', u'new york the dollar', u'the economy has started', u'the federal reserve has', u'the stock market that', u'a annual rate perhaps', u'one of the quintessential', u'in the dollar value', u'the wall street journal', u'a year earlier royal']\n",
      "[u'is expected to be even', u'new york stock exchange already', u'in the stock market where', u'dow jones industrial average lost', u'in the fourth quarter by', u'the nasdaq composite index eased', u'more than a year and', u'the standard poor index edged', u'the nasdaq composite index gained', u'the nasdaq stock market and']\n",
      "[u'the dow jones industrial average fell', u'the dow jones industrial average retook', u'the new york stock exchange composite', u'the new york stock exchange already', u'the dow jones industrial average closed', u'the dow jones industrial average jumped', u'the dow jones industrial average declined', u'the dow jones industrial average down', u'the dow jones industrial average rose', u'the dow jones industrial average from']\n",
      "[]\n"
     ]
    }
   ],
   "source": [
    "while len(index.keys()) > 0:\n",
    "    index_p = {}\n",
    "    for j in index.keys():\n",
    "        if len(index[j]) >= min_threshold_tau:\n",
    "            f[j] = len(index[j])\n",
    "            for k in index[j]:\n",
    "                if k+1 < words_len:\n",
    "                    new_phrase = j + u' ' + all_words[k + 1]\n",
    "                    if new_phrase not in index_p.keys():\n",
    "                        index_p[new_phrase] = [k + 1]\n",
    "                    else:\n",
    "                        index_p[new_phrase].append(k + 1)\n",
    "    index = index_p\n",
    "    print index.keys()[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total phrases above threshold: 2272\n",
      "sample phrases: ['limited', u'an annual', u'the second', u'the most', u'to and the', u'new york stock exchange', 'four', u'unemployment rate', 'asian', u'the last']\n"
     ]
    }
   ],
   "source": [
    "print 'total phrases above threshold:', len(f.keys())\n",
    "print 'sample phrases:', f.keys()[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['limited',\n",
       " u'an annual',\n",
       " u'the second',\n",
       " u'the most',\n",
       " u'to and the',\n",
       " u'new york stock exchange',\n",
       " 'four',\n",
       " u'unemployment rate',\n",
       " 'asian',\n",
       " u'the last',\n",
       " 'increase',\n",
       " u'the president',\n",
       " 'voted',\n",
       " 'under',\n",
       " u'a seasonally',\n",
       " u'and interest',\n",
       " 'worth',\n",
       " 'rescue',\n",
       " 'rise',\n",
       " u'will make',\n",
       " 'every',\n",
       " 'reforms',\n",
       " 'affect',\n",
       " 'school',\n",
       " u'the number of',\n",
       " 'prize',\n",
       " u'a annual rate',\n",
       " 'companies',\n",
       " 'wednesday',\n",
       " 'red',\n",
       " u'against the',\n",
       " 'bernanke',\n",
       " 'force',\n",
       " 'leaders',\n",
       " u'managing director',\n",
       " 'estimates',\n",
       " 'japanese',\n",
       " u'because of',\n",
       " 'second',\n",
       " 'persuade',\n",
       " 'estimated',\n",
       " u'the recession',\n",
       " 'machines',\n",
       " 'even',\n",
       " u'year earlier',\n",
       " u'is it',\n",
       " 'asia',\n",
       " 'spokesman',\n",
       " u'they had',\n",
       " u'the fall',\n",
       " 'above',\n",
       " 'new',\n",
       " 'net',\n",
       " 'ever',\n",
       " 'told',\n",
       " u'a gain',\n",
       " u'and it',\n",
       " u'lower interest',\n",
       " 'whose',\n",
       " u'in many',\n",
       " 'never',\n",
       " u'or to and',\n",
       " 'here',\n",
       " 'reported',\n",
       " 'protection',\n",
       " 'china',\n",
       " u'and in',\n",
       " u'compared to',\n",
       " u'starting to',\n",
       " 'auction',\n",
       " 'study',\n",
       " 'climbed',\n",
       " 'reports',\n",
       " 'credit',\n",
       " 'analysts',\n",
       " 'changes',\n",
       " 'campaign',\n",
       " 'reagan',\n",
       " 'highly',\n",
       " 'total',\n",
       " 'unit',\n",
       " 'would',\n",
       " u'said it',\n",
       " 'negative',\n",
       " 'call',\n",
       " 'until',\n",
       " u'of in',\n",
       " 'composite',\n",
       " 'yahoo',\n",
       " u'nasdaq stock',\n",
       " u'as investors',\n",
       " u'with an',\n",
       " 'hold',\n",
       " 'me',\n",
       " 'rights',\n",
       " 'work',\n",
       " u'the financial markets',\n",
       " u'dow jones industrial average',\n",
       " u'or a',\n",
       " u'first time',\n",
       " u'central bank',\n",
       " u'number of companies',\n",
       " 'my',\n",
       " u'economy the',\n",
       " 'give',\n",
       " 'household',\n",
       " 'india',\n",
       " u'can be',\n",
       " 'currency',\n",
       " u'the dollar',\n",
       " u'in the second',\n",
       " 'want',\n",
       " u'to start',\n",
       " u'of the year',\n",
       " 'risk',\n",
       " 'end',\n",
       " 'recovery',\n",
       " 'thing',\n",
       " 'provide',\n",
       " u'to keep',\n",
       " 'far',\n",
       " u'nasdaq composite index',\n",
       " 'widespread',\n",
       " u'has been',\n",
       " u'have to',\n",
       " 'after',\n",
       " 'modest',\n",
       " 'president',\n",
       " 'law',\n",
       " u'which has',\n",
       " 'attempt',\n",
       " 'third',\n",
       " 'amid',\n",
       " 'things',\n",
       " u'point of',\n",
       " 'democratic',\n",
       " 'order',\n",
       " 'operations',\n",
       " u'in november',\n",
       " 'over',\n",
       " 'fall',\n",
       " 'expects',\n",
       " u'it would',\n",
       " u'rate cut',\n",
       " 'london',\n",
       " 'japan',\n",
       " u'as many',\n",
       " 'mayor',\n",
       " 'before',\n",
       " u'the treasury',\n",
       " u'capital markets',\n",
       " 'imf',\n",
       " 'personal',\n",
       " 'fix',\n",
       " 'expectations',\n",
       " 'better',\n",
       " 'ims',\n",
       " 'production',\n",
       " u'most of',\n",
       " 'weeks',\n",
       " 'then',\n",
       " 'them',\n",
       " u'is one of the',\n",
       " u'the issue',\n",
       " u'to do',\n",
       " 'weakness',\n",
       " 'nobel',\n",
       " 'they',\n",
       " 'silver',\n",
       " 'bank',\n",
       " u'the university of',\n",
       " 'represents',\n",
       " u'is likely to',\n",
       " 'cftc',\n",
       " u'a lot',\n",
       " 'each',\n",
       " u'the central bank',\n",
       " u'is to',\n",
       " 'bond',\n",
       " u'or to',\n",
       " 'financial',\n",
       " u'that have',\n",
       " u'at its',\n",
       " 'series',\n",
       " u'york stock',\n",
       " 'used',\n",
       " 'trading',\n",
       " u'on thursday',\n",
       " u'have a',\n",
       " u'the nasdaq stock market',\n",
       " u'they have',\n",
       " 'laid',\n",
       " u'of dollars',\n",
       " u'to reduce',\n",
       " 'got',\n",
       " 'strategist',\n",
       " 'independence',\n",
       " u'could be',\n",
       " 'turning',\n",
       " u'raising the',\n",
       " 'given',\n",
       " u'chairman of the',\n",
       " 'standard',\n",
       " 'estimate',\n",
       " u'the third',\n",
       " u'federal reserve board',\n",
       " 'created',\n",
       " 'starts',\n",
       " 'days',\n",
       " u'to find',\n",
       " u'banks to',\n",
       " u'this year',\n",
       " u'index fell',\n",
       " 'medicaid',\n",
       " 'already',\n",
       " 'economists',\n",
       " u'before the',\n",
       " 'primary',\n",
       " u'a strong',\n",
       " u'year to',\n",
       " 'another',\n",
       " u'the global',\n",
       " 'electronic',\n",
       " 'households',\n",
       " 'service',\n",
       " u'how the',\n",
       " 'top',\n",
       " u'in the latest',\n",
       " 'needed',\n",
       " u'was a',\n",
       " 'rates',\n",
       " 'too',\n",
       " u'such as',\n",
       " 'percentage',\n",
       " 'john',\n",
       " u'the nasdaq',\n",
       " u'to take',\n",
       " 'ceiling',\n",
       " u'of a',\n",
       " u'million in',\n",
       " 'took',\n",
       " u'an average',\n",
       " 'somewhat',\n",
       " u'a little',\n",
       " u'poor index',\n",
       " u'rates that',\n",
       " u'and many',\n",
       " 'target',\n",
       " 'showed',\n",
       " 'likely',\n",
       " u'which is',\n",
       " 'project',\n",
       " 'street',\n",
       " u'share of',\n",
       " 'minus',\n",
       " 'acquisition',\n",
       " u'the year',\n",
       " 'boston',\n",
       " u'price increases',\n",
       " u'during the',\n",
       " u'that are',\n",
       " 'seen',\n",
       " 'seem',\n",
       " 'relatively',\n",
       " u'most of the',\n",
       " 'strength',\n",
       " u'ben bernanke',\n",
       " u'policy makers',\n",
       " 'indexes',\n",
       " 'materials',\n",
       " u'that was',\n",
       " u'and some',\n",
       " u'when the',\n",
       " u'a larger',\n",
       " 'though',\n",
       " u'interest rate',\n",
       " u'want to',\n",
       " u'the period',\n",
       " u'jones industrial',\n",
       " 'germany',\n",
       " u'price index',\n",
       " 'consumers',\n",
       " 'medical',\n",
       " 'flow',\n",
       " u'lower interest rates',\n",
       " u'to pay',\n",
       " u'we are',\n",
       " 'competitors',\n",
       " 'points',\n",
       " 'tech',\n",
       " u'increase in',\n",
       " u'the economy has',\n",
       " 'consumer',\n",
       " 'dow',\n",
       " 'came',\n",
       " 'reserve',\n",
       " u'if you',\n",
       " u'economy has',\n",
       " u'ago the',\n",
       " 'ending',\n",
       " 'gauge',\n",
       " u'is up',\n",
       " 'just',\n",
       " 'yen',\n",
       " u'of bonds',\n",
       " u'demand for',\n",
       " u'interest rates',\n",
       " 'bush',\n",
       " 'announce',\n",
       " u'rose in',\n",
       " 'do',\n",
       " 'exports',\n",
       " 'despite',\n",
       " 'report',\n",
       " u'that means',\n",
       " 'volatility',\n",
       " u'at least',\n",
       " 'bay',\n",
       " 'bad',\n",
       " u'royal bank',\n",
       " 'release',\n",
       " u'into a',\n",
       " u'million or',\n",
       " u'surge in',\n",
       " 'secretary',\n",
       " u'issue of',\n",
       " 'testing',\n",
       " u'when it',\n",
       " 'result',\n",
       " u'year but',\n",
       " u'last year',\n",
       " 'best',\n",
       " u'came in',\n",
       " 'said',\n",
       " u'bonds and',\n",
       " 'away',\n",
       " u'to in',\n",
       " 'pressures',\n",
       " u'years the',\n",
       " u'the fourth',\n",
       " u'budget deficit',\n",
       " u'after a',\n",
       " u'the new york',\n",
       " 'we',\n",
       " 'terms',\n",
       " 'wo',\n",
       " 'weak',\n",
       " 'however',\n",
       " 'southeast',\n",
       " 'harvard',\n",
       " 'recent',\n",
       " u'the commerce department',\n",
       " 'news',\n",
       " 'debt',\n",
       " 'country',\n",
       " 'against',\n",
       " 'planned',\n",
       " u'president of',\n",
       " u'the index',\n",
       " 'appeared',\n",
       " 'had',\n",
       " u'and they',\n",
       " u'a result',\n",
       " u'of their',\n",
       " u'the canadian',\n",
       " 'trust',\n",
       " 'conference',\n",
       " 'jobless',\n",
       " 'basis',\n",
       " 'holdings',\n",
       " 'three',\n",
       " 'been',\n",
       " 'quickly',\n",
       " 'commission',\n",
       " 'much',\n",
       " 'interest',\n",
       " u'it was a',\n",
       " 'expected',\n",
       " u'the bill',\n",
       " u'the biggest',\n",
       " 'life',\n",
       " 'families',\n",
       " 'mci',\n",
       " u'for the first time',\n",
       " 'republicans',\n",
       " 'lift',\n",
       " u'the federal',\n",
       " u'they do',\n",
       " u'all the',\n",
       " 'commerce',\n",
       " 'has',\n",
       " u'the amount',\n",
       " 'economies',\n",
       " 'air',\n",
       " 'near',\n",
       " 'teachers',\n",
       " 'aid',\n",
       " u'the dow jones industrial',\n",
       " 'employees',\n",
       " 'seven',\n",
       " 'economics',\n",
       " 'mexico',\n",
       " 'is',\n",
       " 'it',\n",
       " 'experts',\n",
       " 'in',\n",
       " 'if',\n",
       " 'grown',\n",
       " u'some of the',\n",
       " 'suggest',\n",
       " 'make',\n",
       " 'several',\n",
       " 'european',\n",
       " 'independent',\n",
       " u'nasdaq stock market',\n",
       " 'hand',\n",
       " u'people are',\n",
       " u'risk of',\n",
       " 'opportunity',\n",
       " 'kept',\n",
       " 'programs',\n",
       " u'the stock',\n",
       " 'warburg',\n",
       " 'savings',\n",
       " 'client',\n",
       " u'end of',\n",
       " 'claims',\n",
       " 'the',\n",
       " 'corporate',\n",
       " 'investments',\n",
       " 'left',\n",
       " 'seasonal',\n",
       " 'proposed',\n",
       " 'unemployment',\n",
       " u'think of',\n",
       " 'purchasing',\n",
       " 'human',\n",
       " 'yet',\n",
       " 'previous',\n",
       " 'buyers',\n",
       " u'now is',\n",
       " 'ease',\n",
       " u'increase the',\n",
       " 'royal',\n",
       " u'have the',\n",
       " 'board',\n",
       " 'save',\n",
       " 'gave',\n",
       " u'rate in',\n",
       " u'economist at',\n",
       " 'possible',\n",
       " u'of the world',\n",
       " u'about of',\n",
       " u'that a',\n",
       " 'advanced',\n",
       " 'desire',\n",
       " u'the end of',\n",
       " 'specific',\n",
       " u'price of',\n",
       " u'their bills',\n",
       " 'security',\n",
       " u'on a',\n",
       " u'reserve board',\n",
       " u'york the',\n",
       " 'right',\n",
       " 'old',\n",
       " 'deal',\n",
       " 'people',\n",
       " u'vice president',\n",
       " 'offer',\n",
       " 'election',\n",
       " 'securities',\n",
       " u'that many',\n",
       " 'for',\n",
       " u'a major',\n",
       " 'comments',\n",
       " 'everything',\n",
       " 'trustees',\n",
       " u'the standard poor',\n",
       " u'if they',\n",
       " u'in fact',\n",
       " u'may be',\n",
       " 'core',\n",
       " u'look at',\n",
       " 'deflation',\n",
       " 'discount',\n",
       " 'corp',\n",
       " u'stock exchange',\n",
       " 'losing',\n",
       " u'following the',\n",
       " 'attacks',\n",
       " u'in the market',\n",
       " 'dollars',\n",
       " 'months',\n",
       " u'in september',\n",
       " 'efforts',\n",
       " 'slightly',\n",
       " 'consulting',\n",
       " 'managers',\n",
       " u'it had',\n",
       " 'civil',\n",
       " u'to more',\n",
       " 'down',\n",
       " u'it has',\n",
       " u'on its',\n",
       " u'continued to',\n",
       " u'in washington',\n",
       " u'but not',\n",
       " 'support',\n",
       " 'initial',\n",
       " 'legislation',\n",
       " 'payment',\n",
       " u'in interest rates',\n",
       " u'whether the',\n",
       " 'way',\n",
       " u'for a',\n",
       " u'at a annual',\n",
       " 'was',\n",
       " 'war',\n",
       " 'lowest',\n",
       " 'head',\n",
       " u'the consensus',\n",
       " 'form',\n",
       " 'manufacturers',\n",
       " u'of them',\n",
       " u'from an',\n",
       " 'differences',\n",
       " 'ford',\n",
       " u'the pace',\n",
       " 'lows',\n",
       " u'prices on',\n",
       " 'analyst',\n",
       " u'familiar with',\n",
       " u'have also',\n",
       " 'tell',\n",
       " u'to billion',\n",
       " u'likely to',\n",
       " u'prices of',\n",
       " u'new york stock',\n",
       " 'evidence',\n",
       " u'the only',\n",
       " u'on the nasdaq',\n",
       " 'no',\n",
       " 'stake',\n",
       " 'when',\n",
       " u'the financial',\n",
       " u'with its',\n",
       " u'the white house',\n",
       " 'test',\n",
       " u'out of',\n",
       " 'welcome',\n",
       " 'fell',\n",
       " u'insurance companies',\n",
       " 'weekend',\n",
       " 'billion',\n",
       " u'but it is',\n",
       " u'the nasdaq composite',\n",
       " 'faster',\n",
       " 'filing',\n",
       " 'vietnam',\n",
       " u'third quarter',\n",
       " 'time',\n",
       " 'push',\n",
       " 'serious',\n",
       " 'profits',\n",
       " u'the national',\n",
       " u'second quarter',\n",
       " 'global',\n",
       " 'focus',\n",
       " 'manager',\n",
       " 'battle',\n",
       " u'and a',\n",
       " 'william',\n",
       " u'million or cents a',\n",
       " u'the university',\n",
       " u'a increase',\n",
       " u'investors are',\n",
       " 'finally',\n",
       " 'weakening',\n",
       " 'marks',\n",
       " u'value of',\n",
       " 'advantage',\n",
       " 'ultrashort',\n",
       " u'cut in',\n",
       " u'fed officials',\n",
       " u'is still',\n",
       " u'financial crisis',\n",
       " 'level',\n",
       " 'did',\n",
       " 'standards',\n",
       " 'p',\n",
       " u'treasury secretary',\n",
       " u'show that',\n",
       " u'york stock exchange',\n",
       " u'up from',\n",
       " 'slower',\n",
       " u'stocks rose',\n",
       " 'sign',\n",
       " 'cost',\n",
       " u'the cftc',\n",
       " u'it is',\n",
       " u'in the stock',\n",
       " u'than of',\n",
       " u'a annual',\n",
       " 'shares',\n",
       " 'current',\n",
       " u'oil prices',\n",
       " u'are likely to',\n",
       " 'international',\n",
       " 'falling',\n",
       " 'boost',\n",
       " u'chairman ben bernanke',\n",
       " 'transportation',\n",
       " u'is expected to',\n",
       " u'concern that',\n",
       " u'in its',\n",
       " 'agreement',\n",
       " u'by a',\n",
       " u'million or cents',\n",
       " 'alone',\n",
       " 'change',\n",
       " 'canadian',\n",
       " u'economy is',\n",
       " u'are likely',\n",
       " u'the imf',\n",
       " u'a lot of',\n",
       " u'capital gains',\n",
       " 'extra',\n",
       " 'lending',\n",
       " u'is that',\n",
       " u'across the',\n",
       " u'according to',\n",
       " u'or that',\n",
       " u'there is no',\n",
       " 'crisis',\n",
       " 'market',\n",
       " u'as the economy',\n",
       " u'the other',\n",
       " 'working',\n",
       " u'familiar with the',\n",
       " 'positive',\n",
       " u'goods and',\n",
       " u'rate the',\n",
       " 'francs',\n",
       " u'support for',\n",
       " u'but that',\n",
       " u'points to',\n",
       " 'today',\n",
       " 'commissioner',\n",
       " 'sessions',\n",
       " u'of million',\n",
       " u'producer price',\n",
       " u'friday the',\n",
       " u'the financial crisis',\n",
       " 'cautious',\n",
       " 'cases',\n",
       " 'effort',\n",
       " 'states',\n",
       " 'valued',\n",
       " u'economic crisis',\n",
       " u'in new york the',\n",
       " u'to million',\n",
       " u'the new',\n",
       " u'that the federal reserve',\n",
       " u'in part',\n",
       " 'can',\n",
       " 'growing',\n",
       " 'making',\n",
       " u'in new york',\n",
       " u'at the end of',\n",
       " 'citizens',\n",
       " u'in which',\n",
       " 'december',\n",
       " u'index of',\n",
       " 'dropped',\n",
       " u'traded at',\n",
       " u'or the',\n",
       " u'retail sales',\n",
       " u'a good',\n",
       " u'from a',\n",
       " 'means',\n",
       " u'the terms',\n",
       " u'down from last',\n",
       " u'outlook for',\n",
       " 'fourth',\n",
       " u'to the',\n",
       " 'economy',\n",
       " 'product',\n",
       " 'information',\n",
       " 'critics',\n",
       " 'use',\n",
       " 'mac',\n",
       " u'is expected',\n",
       " 'improving',\n",
       " 'such',\n",
       " 'data',\n",
       " 'grow',\n",
       " 'man',\n",
       " u'that the',\n",
       " 'natural',\n",
       " 'adelphia',\n",
       " u'the economy',\n",
       " 'futures',\n",
       " 'borrowing',\n",
       " 'so',\n",
       " 'deposit',\n",
       " u'annual rate of',\n",
       " u'decline in',\n",
       " u'led by',\n",
       " u'the session',\n",
       " 'raised',\n",
       " 'years',\n",
       " 'hiring',\n",
       " u'the highest',\n",
       " 'cuts',\n",
       " 'argued',\n",
       " u'on friday',\n",
       " 'still',\n",
       " u'believe the',\n",
       " 'group',\n",
       " 'how',\n",
       " u'at a annual rate',\n",
       " u'investment in',\n",
       " u'consumer credit',\n",
       " 'policy',\n",
       " 'main',\n",
       " 'decades',\n",
       " 'texas',\n",
       " u'is likely',\n",
       " u'morgan stanley',\n",
       " u'in late',\n",
       " 'finance',\n",
       " u'the nation',\n",
       " 'nation',\n",
       " 'records',\n",
       " 'half',\n",
       " 'not',\n",
       " 'now',\n",
       " 'maintaining',\n",
       " 'term',\n",
       " 'name',\n",
       " u'the unemployment',\n",
       " 'january',\n",
       " 'times',\n",
       " 'drop',\n",
       " u'and so',\n",
       " 'quarter',\n",
       " u'the project',\n",
       " 'revised',\n",
       " u'the past three',\n",
       " 'year',\n",
       " 'happen',\n",
       " 'morning',\n",
       " u'of america',\n",
       " 'canada',\n",
       " 'factors',\n",
       " 'profit',\n",
       " u'number of',\n",
       " 'looking',\n",
       " 'investigation',\n",
       " 'internet',\n",
       " 'shows',\n",
       " 'earlier',\n",
       " 'million',\n",
       " 'cut',\n",
       " u'the deal',\n",
       " 'care',\n",
       " 'advance',\n",
       " u'the economic',\n",
       " 'turn',\n",
       " u'points or to',\n",
       " 'place',\n",
       " 'think',\n",
       " 'first',\n",
       " u'that is',\n",
       " u'rose to',\n",
       " u'that it',\n",
       " 'revenue',\n",
       " u'shot up',\n",
       " 'bargaining',\n",
       " u'on the',\n",
       " 'one',\n",
       " u'the outlook',\n",
       " 'americans',\n",
       " 'directly',\n",
       " 'vote',\n",
       " u'have been',\n",
       " 'open',\n",
       " u'that in',\n",
       " 'city',\n",
       " 'little',\n",
       " u'a decline in',\n",
       " u'the and',\n",
       " 'trillion',\n",
       " u'a month',\n",
       " 'anyone',\n",
       " 'indicate',\n",
       " 'white',\n",
       " 'frame',\n",
       " u'the end',\n",
       " 'mostly',\n",
       " 'that',\n",
       " 'season',\n",
       " 'alan',\n",
       " u'was at',\n",
       " 'released',\n",
       " u'the recovery',\n",
       " 'than',\n",
       " 'wide',\n",
       " 'procter',\n",
       " u'a professor',\n",
       " 'future',\n",
       " 'venture',\n",
       " 'were',\n",
       " 'and',\n",
       " u'mortgage rates',\n",
       " u'said in a',\n",
       " 'san',\n",
       " 'investors',\n",
       " u'the study',\n",
       " u'three years',\n",
       " 'turned',\n",
       " 'argument',\n",
       " 'say',\n",
       " u'the same',\n",
       " u'or cents a',\n",
       " 'saw',\n",
       " 'any',\n",
       " u'concerns about',\n",
       " u'as much as',\n",
       " 'offering',\n",
       " 'northview',\n",
       " 'equipment',\n",
       " u'the bush administration',\n",
       " 'potential',\n",
       " 'take',\n",
       " 'online',\n",
       " 'performance',\n",
       " u'close to',\n",
       " 'begin',\n",
       " u'this past',\n",
       " 'price',\n",
       " u'market the',\n",
       " 'paid',\n",
       " u'is expected to be',\n",
       " 'america',\n",
       " u'the late',\n",
       " u'economy and',\n",
       " u'and other',\n",
       " 'especially',\n",
       " 'average',\n",
       " 'later',\n",
       " 'steady',\n",
       " 'sale',\n",
       " 'federal',\n",
       " u'end of the',\n",
       " u'signs of',\n",
       " 'senior',\n",
       " 'typically',\n",
       " 'liquidity',\n",
       " 'shot',\n",
       " 'show',\n",
       " 'german',\n",
       " u'the quarter',\n",
       " 'aggressive',\n",
       " u'a big',\n",
       " 'slow',\n",
       " u'the nasdaq composite index',\n",
       " 'office',\n",
       " 'behind',\n",
       " 'only',\n",
       " 'going',\n",
       " 'treasury',\n",
       " u'rates in',\n",
       " 'get',\n",
       " 'contracts',\n",
       " 'assistant',\n",
       " u'social security',\n",
       " u'that they',\n",
       " 'employers',\n",
       " u'under the',\n",
       " 'nearly',\n",
       " u'about a',\n",
       " 'prime',\n",
       " u'a share',\n",
       " u'this was',\n",
       " 'yield',\n",
       " 'worried',\n",
       " u'dollar and',\n",
       " u'a sharp',\n",
       " 'worries',\n",
       " 'where',\n",
       " u'closed at',\n",
       " u'annual rate',\n",
       " u'according to a',\n",
       " 'stanley',\n",
       " 'concern',\n",
       " 'mortgage',\n",
       " 'managing',\n",
       " 'jumped',\n",
       " u'rather than',\n",
       " 'forecast',\n",
       " u'because the',\n",
       " u'the nasdaq stock',\n",
       " 'enough',\n",
       " 'bureau',\n",
       " 'between',\n",
       " 'import',\n",
       " 'reading',\n",
       " 'across',\n",
       " 'jobs',\n",
       " 'august',\n",
       " u'the first quarter',\n",
       " 'come',\n",
       " u'as a result',\n",
       " 'many',\n",
       " 'region',\n",
       " 'recessions',\n",
       " 'contract',\n",
       " 's',\n",
       " u'labor department',\n",
       " 'traded',\n",
       " 'comes',\n",
       " 'among',\n",
       " u'for the second',\n",
       " 'period',\n",
       " 'considering',\n",
       " u'it was',\n",
       " 'west',\n",
       " 'airlines',\n",
       " 'mark',\n",
       " u'are the',\n",
       " 'wants',\n",
       " u'that the economy is',\n",
       " 'direction',\n",
       " 'sectors',\n",
       " 'offered',\n",
       " u'companies and',\n",
       " u'in what',\n",
       " u'days of',\n",
       " 'former',\n",
       " 'those',\n",
       " 'case',\n",
       " 'these',\n",
       " u'be the',\n",
       " 'cash',\n",
       " 'policies',\n",
       " 'situation',\n",
       " 'margin',\n",
       " 'trader',\n",
       " u'imports of',\n",
       " 'eventually',\n",
       " 'according',\n",
       " 'middle',\n",
       " u'or at',\n",
       " 'technology',\n",
       " 'different',\n",
       " 'author',\n",
       " 'pay',\n",
       " u'she says',\n",
       " 'same',\n",
       " u'fourth quarter',\n",
       " u'than billion',\n",
       " u'in december',\n",
       " u'decline in the',\n",
       " 'struggling',\n",
       " u'the number',\n",
       " 'week',\n",
       " 'oil',\n",
       " 'edged',\n",
       " u'about to',\n",
       " 'singapore',\n",
       " u'billion in the',\n",
       " 'director',\n",
       " u'is one',\n",
       " 'anticipated',\n",
       " u'be a',\n",
       " u'months of',\n",
       " u'the end of the',\n",
       " u'director of',\n",
       " u'the market is',\n",
       " 'roughly',\n",
       " 'mortgages',\n",
       " 'without',\n",
       " u'become a',\n",
       " u'department said',\n",
       " u'about the',\n",
       " 'easing',\n",
       " 'laboratories',\n",
       " u'to be the',\n",
       " 'gdp',\n",
       " 'gates',\n",
       " u'but there',\n",
       " 'being',\n",
       " 'money',\n",
       " 'weekly',\n",
       " u'of the',\n",
       " 'thinking',\n",
       " 'rose',\n",
       " 'seems',\n",
       " ...]"
      ]
     },
     "execution_count": 333,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f.keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Quality Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "phrases = []\n",
    "for w in f.keys():\n",
    "    if len(w.split(u' ')) > 1:\n",
    "        phrases.append(w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 2 words and above phrases: 972\n",
      "sample: [u'an annual', u'the second', u'the most', u'to and the', u'new york stock exchange', u'unemployment rate', u'the last', u'the president', u'a seasonally', u'and interest']\n"
     ]
    }
   ],
   "source": [
    "print 'total 2 words and above phrases:', len(phrases)\n",
    "print 'sample:', phrases[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Concordance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "PMI = {}\n",
    "PLK = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def prob_u(phrase, f):\n",
    "    words = phrase.split(u' ')\n",
    "    denom = 0\n",
    "    for j in words:\n",
    "        denom += f[word]\n",
    "    return f[phrase]*1. / denom"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def cal_PMI_PLK(phrase, f, pmi, plk):\n",
    "    words = phrase.split(u' ')\n",
    "    prob_whole = prob_u(phrase, f)\n",
    "    if len(words) > 2:\n",
    "        min_mutual_info = float('inf')\n",
    "        best_u_left = None\n",
    "        best_u_right = None\n",
    "        for i in range(1, len(words)):\n",
    "            u_left = u' '.join(words[:i])\n",
    "            u_right = u' '.join(words[i:])\n",
    "            info = math.log(prob_whole * 1. / (prob_u(u_left, f) * prob_u(u_right, f)))\n",
    "            if info < min_mutual_info:\n",
    "                min_mutual_info = info\n",
    "                best_u_left = u_left\n",
    "                best_u_right = u_right\n",
    "    else:\n",
    "        best_u_left = words[0]\n",
    "        best_u_right = words[1]\n",
    "    \n",
    "    pmi[phrase] = math.log(prob_whole * 1. / (prob_u(best_u_left, f) * prob_u(best_u_right, f)))\n",
    "    plk[phrase] = prob_whole * math.log(prob_whole * 1. / (prob_u(best_u_left, f) * prob_u(best_u_right, f)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for p in phrases:\n",
    "    cal_PMI_PLK(p, f, PMI, PLK)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "972\n",
      "[u'last month', u'a record', u'to say', u'second quarter', u'an annual', u'to have', u'the second', u'the most', u'the wall street', u'unemployment rate for']\n",
      "-4.31255287835\n",
      "-11.2126374837\n"
     ]
    }
   ],
   "source": [
    "print len(PMI.keys())\n",
    "print PMI.keys()[:10]\n",
    "print PMI[PMI.keys()[0]]\n",
    "print PLK[PLK.keys()[0]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Informativeness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "IDF_phrase = {}\n",
    "IDF_word = {}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total articles: 200\n"
     ]
    }
   ],
   "source": [
    "print 'total articles:', len(articles)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def cal_word_IDF(docs, idf_word):\n",
    "    total_length = len(docs)\n",
    "    for w in all_words:\n",
    "        count = 0\n",
    "        for d in docs:\n",
    "            if w in d:\n",
    "                count += 1\n",
    "        if count > 0:\n",
    "            idf_word[w] = math.log(total_length * 1. / count)\n",
    "        else:\n",
    "            idf_word[w] = 0\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cal_word_IDF(all_articles, IDF_word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['four', 'mirage', 'adjustors', 'increase', 'eligible']\n",
      "2.20727491319\n",
      "5.29831736655\n",
      "5.29831736655\n",
      "1.83258146375\n",
      "4.19970507788\n"
     ]
    }
   ],
   "source": [
    "print IDF_word.keys()[:5]\n",
    "index = IDF_word.keys()[:5]\n",
    "for i in index:\n",
    "    print IDF_word[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cal_phrase_IDF(phrases, word_idf, phrase_idf):\n",
    "    for p in phrases:\n",
    "        idf = 0\n",
    "        words = p.split(u' ')\n",
    "        for w in words:\n",
    "            idf += word_idf[w]\n",
    "        phrase_idf[p] = idf * 1. / len(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 347,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cal_phrase_IDF(phrases, IDF_word, IDF_phrase)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'last month', u'a record', u'to say', u'second quarter', u'an annual', u'to have', u'the second', u'the most', u'the wall street', u'unemployment rate for']\n",
      "1.39233065174\n",
      "1.22699477358\n",
      "0.939721899936\n",
      "2.08669388478\n",
      "1.46403681254\n",
      "0.301950401523\n",
      "1.10614372751\n",
      "0.70575480513\n",
      "1.58894551262\n",
      "1.36355710562\n"
     ]
    }
   ],
   "source": [
    "print IDF_phrase.keys()[:10]\n",
    "for p in IDF_phrase.keys()[:10]:\n",
    "    print IDF_phrase[p]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = {}\n",
    "for p in phrases:\n",
    "    data[p] = []\n",
    "    data[p].append(float(PMI[p]))\n",
    "    data[p].append(float(PLK[p]))\n",
    "    data[p].append(float(IDF_phrase[p]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'last month', u'a record', u'to say', u'second quarter', u'an annual', u'to have', u'the second', u'the most', u'the wall street', u'unemployment rate for', u'to give', u'unemployment rate', u'chairman of', u'in what', u'the last', u'trade deficit', u'companies that', u'million or cents a', u'continues to', u'a increase', u'meanwhile the', u'said in', u'sign that', u'said it', u'the president', u'of in', u'ability to', u'is no', u'a seasonally', u'and interest', u'dow jones industrial average', u'much of', u'bush administration', u'years ago', u'warburg pincus', u'will make', u'funds and', u'fed officials', u'they were', u'is still', u'financial crisis', u'more than of', u'than a', u'fed chairman', u'over the', u'to and', u'treasury secretary', u'show that', u'york a', u'against the', u'york stock exchange', u'up from', u'wall street', u'due to', u'the global', u'or to the', u'stocks rose', u'when he', u'year according', u'declined to', u'of all', u'the cftc', u'that are', u'the recession', u'about of', u'the period', u'the first time', u'program to', u'the law', u'used to', u'in the stock', u'economy has', u'or a', u'like the', u'they had', u'of some', u'first time', u'the fall', u'are likely to', u'that the federal', u'and a', u'chairman ben bernanke', u'past three', u'surge in', u'such a', u'and it', u'lower interest', u'in many', u'rise in', u'concern that', u'or to and', u'in its', u'by a', u'the average', u'the index', u'to a', u'million or cents', u'and with', u'up their', u'and in', u'the government', u'compared to', u'this is', u'starting to', u'below the', u'more than a', u'to people', u'economy is', u'the fed has', u'a billion', u'the market', u'professor at', u'when a', u'are likely', u'some of the', u'new york the', u'a sign', u'rate of', u'the white', u'capital gains', u'amount of', u'yield on', u'if we', u'now is', u'is that', u'agreed to', u'chairman ben', u'according to', u'reserve board', u'inflation is', u'of it', u'there is no', u'up to', u'as the economy', u'as much', u'the other', u'from in', u'following the', u'familiar with the', u'year and', u'goods and', u'to more', u'up the', u'at a annual', u'said the', u'rate the', u'economic recovery', u'support for', u'but that', u'whether the', u'as of', u'year end', u'but it', u'there are', u'increase of', u'of million', u'bond prices', u'producer price', u'half of', u'friday the', u'the financial crisis', u'nasdaq stock', u'as investors', u'years old', u'the national', u'central bankers', u'had been', u'new york a', u'can be', u'efforts to', u'the first', u'how much', u'economic crisis', u'in new york the', u'the new', u'chief economist', u'the financial markets', u'that the federal reserve', u'the country', u'a annual', u'in part', u'would have', u'central bank', u'gains have', u'stock prices', u'are expected', u'in new york', u'the imf', u'a lot of', u'at the end of', u'of million or', u'the deficit', u'is not', u'in which', u'energy prices', u'bank of', u'index of', u'to million', u'interest rates and', u'the dollar', u'in the second', u'about million', u'to start', u'traded at', u'of the year', u'or the', u'retail sales', u'a good', u'from the', u'from a', u'to keep', u'the standard', u'the terms', u'down from last', u'outlook for', u'is likely', u'the bond', u'that the market', u'to the', u'to make', u'nasdaq composite index', u'the dow', u'quarter and', u'of america', u'the bank', u'have to', u'four years', u'that the economy', u'a year', u'end of', u'rose points', u'is expected', u'to help', u'business investment', u'as well', u'of economic', u'that the', u'better than', u'the rise', u'billion in', u'the start', u'the economy and', u'the economy', u'vice president of', u'the two', u'cut in', u'annual rate of', u'decline in', u'mutual funds', u'in november', u'led by', u'analysts said', u'the session', u'the unemployment rate', u'to keep the', u'the markets', u'plan to', u'it would', u'rate cut', u'dollar and', u'fell to', u'on friday', u'as many', u'the standard poor index', u'believe the', u'markets and', u'the treasury', u'although the', u'capital markets', u'the mortgage', u'enough to', u'part of', u'the decline', u'keep the', u'up in', u'the dow jones', u'the yen', u'at a annual rate', u'investment in', u'at the end', u'consumer credit', u'continue to', u'most of', u'the number of', u'for the first', u'but they', u'morgan stanley', u'mechanism design', u'is one of the', u'in late', u'the issue', u'to do', u'the nation', u'in december', u'a few', u'known as', u'stock market', u'weakness in', u'to go', u'but there', u'only a', u'nasdaq composite', u'the budget', u'the university of', u'an index', u'to see', u'declines in', u'japan and', u'the unemployment', u'years the', u'in may', u'a lot', u'the senate', u'is to', u'or to', u'it could', u'industrial average', u'at its', u'with the', u'york stock', u'of these', u'the past three', u'over the next', u'natural gas', u'the state', u'cents or to', u'of the most', u'on thursday', u'have a', u'the nasdaq stock market', u'effort to', u'number of', u'they have', u'of dollars', u'to reduce', u'this past', u'compared with a', u's p', u'could be', u'one of', u'up for', u'raising the', u'chairman of the', u'professor of', u'and analysts', u'the new york stock exchange', u'the deal', u'past two', u'the region', u'the city', u'yesterday as', u'with an', u'at billion', u'federal reserve board', u'next month', u'reduce the', u'economy the', u'to find', u'the group', u'monetary policy', u'points or to', u'banks to', u'we have', u'this year', u'the and', u'in annual', u'of government', u'that is', u'their own', u'that it', u'of the world', u'shot up', u'the second quarter', u'before the', u'on the', u'since the', u'a strong', u'the outlook', u'the central bank', u'year to', u'that have', u'have been', u'and other', u'that in', u'to push', u'as they', u'a drop', u'or more', u'a decline in', u'index fell', u'traders said', u'a month', u'in the latest', u'to get', u'was a', u'part of a', u'managing director', u'such as', u'rates that', u'the nasdaq', u'to take', u'the risk', u'few weeks', u'the end', u'of a', u'ben bernanke', u'at a', u'million in', u'of the new', u'was at', u'than in', u'an average', u'drop in', u'in august', u'a little', u'poor index', u'the economy is', u'a new', u'was the', u'he said', u'and many', u'a professor', u'commerce department', u'year ago', u'which is', u'as the', u'the dow jones industrial average', u'the past', u'share of', u'in the first quarter', u'purchasing managers', u'to continue', u'mortgage rates', u'said in a', u'annual rate', u'the study', u'the year', u'it was a', u'price increases', u'during the', u'the trade', u'down from', u'a gain', u'year according to', u'the same', u'he says', u'and for', u'concerns about', u'most of the', u'as much as', u'in the first', u'dollar was', u'policy makers', u'the bush administration', u'have the', u'that was', u'and some', u'been a', u'when the', u'the end of', u'of canada', u'the public', u'a larger', u'are expected to', u'interest rate', u'want to', u'look at', u'university of', u'jones industrial', u'the job', u'last week', u'the federal reserve', u'as a', u'point of', u'the overall', u'the late', u'economy and', u'year the', u'prices on', u'more than', u'the japanese', u'pace of', u'prices of', u'increase the', u'lower interest rates', u'to pay', u'we are', u'companies to', u'the day', u'end of the', u'increase in', u'companies and', u'in june', u'the economy has', u'has a', u'fed has', u'by the', u'if you', u'than of', u'a basis', u'the labor', u'the bush', u'investors are', u'value of', u'food and', u'ago the', u'has become', u'a seasonally adjusted', u'should have', u'many investors', u'two years', u'of and', u'it is', u'the recovery', u'is up', u'the nasdaq composite index', u'the art', u'the current', u'demand for', u'interest rates', u'the economy the', u'oil prices', u'even as', u'rose in', u'to billion in', u'it a', u'the employment', u'the program', u'in one', u'social security', u'that they', u'for each', u'under the', u'about a', u'that means', u'the next', u'is one of', u'a share', u'a number of', u'at least', u'this was', u'the highest', u'the american', u'the jobless', u'royal bank', u'a sharp', u'and energy', u'cents a share', u'into a', u'recession in', u'in the past', u'the economic', u'is at', u'that some', u'issue of', u'some of', u'the report', u'when it', u'year but', u'valued at', u'according to a', u'or cents a', u'last year', u'to their', u'that would', u'is expected to', u'came in', u'bonds and', u'signs of', u'rather than', u'to in', u'the nasdaq stock', u'it also', u'the fourth', u'rates in', u'budget deficit', u'after a', u'the new york', u'market and', u'because it', u'a annual rate', u'increased to', u'between the', u'in their', u'like it', u'the first quarter', u'wall street journal', u'standard poor', u'in april', u'food and energy', u'fell points', u'around the', u'back in', u'as a result', u'mci worldcom', u'in the fourth', u'compared with', u'a more', u'time in', u'she says', u'index rose', u'decline in the', u'has to', u'president of', u'of million or cents', u'to be', u'the market is', u'chief economist at', u'become a', u'confidence in', u'washington the', u'the largest', u'for the second', u'data the', u'the fourth quarter', u'them to', u'and they', u'a result', u'of their', u'the canadian', u'to increase', u'the time', u'of an', u'it was', u'and the', u'the results', u'street journal', u'three months', u'there were', u'first quarter', u'three years', u'are the', u'the world', u'a rise', u'try to', u'to buy', u'the bill', u'the biggest', u'for the', u'one of the', u'for the first time', u'of new', u'head of', u'by the federal', u'trying to', u'to and the', u'the federal', u'they do', u'days of', u'from last', u'composite index', u'seasonally adjusted', u'all the', u'we had', u'the amount', u'through the', u'prices and', u'be the', u'because of', u'into the', u'is it', u'the house', u'has been', u'imports of', u'the gains', u'since april', u'that we', u'or cents', u'is about', u'was up', u'year earlier', u'or at', u'number of companies', u'this week', u'there a', u'the new york stock', u'plans to', u'a high', u'give the', u'federal reserve', u'however the', u'of million or cents a', u'after the', u'of about', u'and more', u'should be', u'the s p', u'than billion', u'series of', u'is likely to', u'and has', u'the number', u'labor department', u'but the', u'a series', u'in new', u'about to', u'the bond market', u'nasdaq stock market', u'billion in the', u'of business', u'points or', u'the third', u'people are', u'is one', u'of money', u'new york', u'that he', u'how the', u'be a', u'months of', u'inflation and', u'the end of the', u'director of', u'the stock', u'the is', u'a large', u'the fed', u'the dollar was', u'to billion in the', u'department said', u'about the', u'over the past', u'that its', u'to be the', u'the commerce department', u'the big', u'the past two', u'even if', u'think of', u'new york stock exchange', u'is the', u'to from', u'this past week', u'market is', u'with a', u'months ago', u'fourth quarter', u'of the', u'even more', u'in when', u'kind of', u'the euro', u'in the fourth quarter', u'the money', u'ahead of', u'risk of', u'of bonds', u'rate for', u'as one', u'recent years', u'consumer prices', u'standard poor index', u'the dow jones industrial', u'rate in', u'in the stock market', u'the economic recovery', u'economist at', u'is a', u'said that', u'the potential', u'cents a', u'the way', u'the numbers', u'in the', u'the housing', u'that a', u'rates to', u'the talks', u'in the economy', u'more than a year', u'credit markets', u'likely to be', u'attempt to', u'a level', u'the wall', u'less than', u'opportunity to', u'the final', u'all of', u'for the year', u'much as', u'price of', u'their bills', u'the quarter', u'on a', u'or that', u'york the', u'financial markets', u'in this', u'would be', u'in the dollar', u'vice president', u'of its', u'of them', u'which has', u'to raise', u'the previous', u'the central', u'have also', u'cents or', u'lot of', u'the s', u'a series of', u'next year', u'the white house', u'it will', u'in october', u'will be', u'of the economy', u'that many', u'and so', u'a major', u'may have', u'the university', u'rates and', u'to an', u'the standard poor', u'at to', u'in fact', u'time to', u'may be', u'market the', u'the news', u'because the', u'than the', u'is that the', u'the week', u'if they', u'instead of', u'million or', u'on tuesday', u'with billion', u'according to the', u'stock exchange', u'across the', u'who are', u'for their', u'the commerce', u'in an', u'of people', u'to its', u'might be', u'late wednesday', u'while the', u'in the market', u'and inflation', u'in interest', u'a rate', u'market that', u'in september', u'of this', u'and that', u'a year earlier', u'in more than', u'it had', u'to think', u'is expected to be', u'close to', u'it has', u'on its', u'the rate', u'continued to', u'in washington', u'freddie mac', u'expected to be', u'past week', u'sign of', u'a decline', u'going to', u'to investors', u'but not', u'rose to', u'pressure on', u'in interest rates', u'the currency', u'bond market', u'for a', u'and interest rates', u'on their', u'a number', u'percentage point', u'dow jones industrial', u'in recent years', u'many economists', u'the consensus', u'surveyed by', u'expected to', u'a bit', u'from an', u'the federal reserve board', u'in more', u'in a', u'the pace', u'the company', u'the labor department', u'price index', u'a big', u'jones industrial average', u'the latest', u'employment report', u'than a year', u'familiar with', u'the trade deficit', u'there is', u'all that', u'to allow', u'in july', u'in recent', u'in and', u'higher interest', u'closed at', u'points to', u'in any', u'labor market', u'they are', u'the reagan', u'white house', u'to billion', u'likely to', u'that the economy is', u'new york stock', u'the board', u'the crash', u'the only', u'health care', u'economic growth', u'cut in the', u'on the nasdaq', u'said a', u'a recession', u'the financial', u'with its', u'in january', u'prices are', u'the securities', u'the recent', u'the wall street journal', u'the firm', u'out of', u'an ounce', u'terms of', u'and new', u'its own', u'of companies', u'quarter of', u'of the fed', u'dow jones', u'for more', u'a year ago', u'more than billion', u'at the', u'and investors', u'insurance companies', u'central banks', u'the stock market', u'but it is', u'the nasdaq composite', u'the price', u'in some', u'the department', u'for the week', u'if the', u'a million', u'sales and', u'that has', u'the project', u'third quarter', u'the mark', u'growth in', u'the capital', u'if it']\n"
     ]
    }
   ],
   "source": [
    "print(data.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "qual_phrase = [u'dow jones industrial average', \n",
    "              u'chairman ben bernanke', \n",
    "              u'economic recovery',\n",
    "              u'business investment', \n",
    "              u'nasdaq composite index', \n",
    "              u'commerce department', \n",
    "              u'interest rates', \n",
    "             u'seasonally adjusted', \n",
    "              u'labor department', \n",
    "              u'central banks']\n",
    "rand_phrase = [u'to billion',\n",
    "              u'and new',\n",
    "              u'but it is',\n",
    "              u'starting to', \n",
    "              u'if we',\n",
    "              u'year and', \n",
    "              u'said in',\n",
    "              u'rise in',\n",
    "              u'or to the',\n",
    "              u'professor at']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_data = []\n",
    "for j in qual_phrase:\n",
    "        train_data.append([1] + data[j])\n",
    "\n",
    "for j in rand_phrase:\n",
    "        train_data.append([0] + data[j])\n",
    "train_data = np.array(train_data)\n",
    "np.random.shuffle(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 361,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20, 4)\n",
      "(20, 3)\n",
      "(20,)\n",
      "[[  0.          -7.07898996  -9.20268695   0.97420664]\n",
      " [  0.          -5.4672159   -2.73360795   2.0227772 ]\n",
      " [  0.          -8.03835707 -12.05753561   0.79992947]\n",
      " [  1.          -4.05352257 -31.21212377   1.18757789]\n",
      " [  1.          -2.31055326  -0.77018442   3.05385007]\n",
      " [  1.          -1.38629436  -0.97040605   3.22675   ]\n",
      " [  1.          -4.76217393  -2.85730436   2.06137202]\n",
      " [  0.          -9.38438779  -4.6921939    0.38739558]\n",
      " [  1.          -3.33220451  -6.33118857   2.23270412]\n",
      " [  0.          -8.46005125  -5.64003417   0.26194802]]\n"
     ]
    }
   ],
   "source": [
    "print(train_data.shape)\n",
    "X = train_data[:, 1:]\n",
    "y = train_data[:, 0]\n",
    "print X.shape\n",
    "print y.shape\n",
    "print train_data[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ -5.9176027   -2.95880135   1.46403681]\n",
      " [ -7.38013109 -14.76026218   1.10614373]\n",
      " [ -8.26719219 -14.88094593   0.70575481]\n",
      " ..., \n",
      " [ -5.76957022  -3.84638014   1.07744386]\n",
      " [ -9.29922998  -9.29922998   0.71606445]\n",
      " [ -9.10817475  -4.55408737   1.08391785]]\n"
     ]
    }
   ],
   "source": [
    "test_data = []\n",
    "test_phrases = []\n",
    "for p in phrases:\n",
    "    if p not in qual_phrase and p not in rand_phrase:\n",
    "        test_data.append(data[p])\n",
    "        test_phrases.append(p)\n",
    "test_data = np.array(test_data)\n",
    "print test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 362,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=3, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_split=1e-07, min_samples_leaf=1,\n",
       "            min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "            n_estimators=10, n_jobs=1, oob_score=False, random_state=0,\n",
       "            verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 362,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "clf = RandomForestClassifier(max_depth=3, n_estimators=10,random_state=0)\n",
    "clf.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[u'an annual', 0.19], [u'the second', 0.14999999999999999], [u'the most', 0.0], [u'to and the', 0.13999999999999999], [u'new york stock exchange', 0.76000000000000001], [u'unemployment rate', 0.77500000000000002], [u'the last', 0.0], [u'the president', 0.13500000000000001], [u'a seasonally', 0.19], [u'and interest', 0.025000000000000001]]\n"
     ]
    }
   ],
   "source": [
    "predictions = []\n",
    "pred_data = clf.predict_proba(test_data)\n",
    "for k in range(len(test_data)):\n",
    "    predictions.append([test_phrases[k], pred_data[k][1]])\n",
    "print(predictions[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best predictions: [[u'managing director', 1.0], [u'central bank', 0.84999999999999998], [u'price increases', 0.90000000000000002], [u'ben bernanke', 1.0], [u'policy makers', 1.0], [u'royal bank', 0.90000000000000002], [u'budget deficit', 0.90000000000000002], [u'their bills', 0.90000000000000002], [u'vice president', 0.98000000000000009], [u'familiar with', 0.90000000000000002]]\n",
      "worst predictions: [[u'an annual', 0.19], [u'the second', 0.14999999999999999], [u'the most', 0.0], [u'to and the', 0.13999999999999999], [u'the last', 0.0], [u'the president', 0.13500000000000001], [u'a seasonally', 0.19], [u'and interest', 0.025000000000000001], [u'a annual rate', 0.11000000000000001], [u'against the', 0.050000000000000003]]\n"
     ]
    }
   ],
   "source": [
    "best_predictions = []\n",
    "worst_predictions = []\n",
    "for p in predictions:\n",
    "    if p[1] >= 0.8:\n",
    "        best_predictions.append(p)\n",
    "    elif p[1] < 0.2:\n",
    "        worst_predictions.append(p)\n",
    "print 'best predictions:', best_predictions[:10]\n",
    "print 'worst predictions:', worst_predictions[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}