industry_classification_pretrained_vectors.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Libs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T00:29:28.872938Z",
     "start_time": "2019-10-08T00:29:20.732902Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize \n",
    "import warnings \n",
    "from scipy import spatial\n",
    "import gensim\n",
    "from gensim.models import Word2Vec \n",
    "# import nltk\n",
    "# nltk.download('punkt')\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "import re\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "porter_stemmer = PorterStemmer()\n",
    "stop_words = set(stopwords.words('english'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T02:46:19.361844Z",
     "start_time": "2019-10-08T02:46:19.356857Z"
    }
   },
   "outputs": [],
   "source": [
    "#pip install stop-words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T00:59:19.344500Z",
     "start_time": "2019-10-08T00:59:17.931186Z"
    }
   },
   "outputs": [],
   "source": [
    "def read_files(file_loc):\n",
    "    df = pd.ExcelFile(file_loc)\n",
    "    df = df.parse('Sheet1')\n",
    "    return df\n",
    "\n",
    "desc = read_files(r\"C:\\Users\\Aishwarya\\Desktop\\ISB task\\OneDrive_1_01-10-2019\\company descriptions.xlsx\")\n",
    "keyw = read_files(r\"C:\\Users\\Aishwarya\\Desktop\\ISB task\\OneDrive_1_01-10-2019\\Industry Segments - Top 10 Keywords.xlsx\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data treatment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T00:59:20.238114Z",
     "start_time": "2019-10-08T00:59:19.346405Z"
    }
   },
   "outputs": [],
   "source": [
    "desc['company_description'] = desc['company_description'].str.strip()\n",
    "desc['company_short_description'] = desc['company_short_description'].str.strip()\n",
    "\n",
    "\n",
    "desc['company_description'].replace( { r'[^A-Za-z ]+' : '' }, inplace= True, regex = True)\n",
    "desc['company_short_description'].replace( { r'[^A-Za-z ]+' : '' }, inplace= True, regex = True)\n",
    "desc = desc.replace(np.nan, '', regex = True) # Replace nans with ''\n",
    "desc['company_description'] = np.where(desc['company_description'] == '', desc['company_short_description'], desc['company_description'])\n",
    "desc['company_description'] = np.where(desc['company_description'].str.len() < 25, desc['company_short_description'] + desc['company_description'], desc['company_description'])\n",
    "\n",
    "desc['company_description'] = desc['company_description'].str.replace('http\\S+|www.\\S+', '', case=False)\n",
    "\n",
    "keyw['Top 10 keywords'].replace( { r'[^A-Za-z ]+' : '' }, inplace= True, regex = True)\n",
    "keyw = keyw.replace(np.nan, '', regex = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load pre-trained vectors from Gensim "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T00:30:00.304448Z",
     "start_time": "2019-10-08T00:29:51.857034Z"
    }
   },
   "outputs": [],
   "source": [
    "from gensim.models import KeyedVectors\n",
    "\n",
    "# Load vectors directly from the file\n",
    "model = KeyedVectors.load_word2vec_format(\"C:\\word2vec\\GoogleNews-vectors-negative300.bin.gz\", binary=True, limit = 250000) # limiting to 250000 words to avoid memory issues and slow runtimes. Library actually has 3 mil words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T01:24:48.780595Z",
     "start_time": "2019-10-08T01:24:48.772613Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_mean_vector(word2vec_model, words):\n",
    "    # remove out-of-vocabulary words\n",
    "    words = [word for word in words if word in word2vec_model.wv.vocab]\n",
    "    if len(words) >= 1:\n",
    "        return np.mean(word2vec_model[words], axis=0)\n",
    "    else:\n",
    "        return []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T02:46:31.190830Z",
     "start_time": "2019-10-08T02:46:31.186838Z"
    }
   },
   "outputs": [],
   "source": [
    "vecs = np.zeros((desc.shape[0],300))\n",
    "err = []\n",
    "\n",
    "for i in range(0,desc.shape[0]):\n",
    "    comp_desc = desc.iloc[i,2]\n",
    "\n",
    "    data = []\n",
    "    # tokenize the sentence into words \n",
    "    for j in word_tokenize(comp_desc):\n",
    "        if j not in stop_words:\n",
    "            data.append(j.lower())\n",
    "    try:\n",
    "        vecs[i] = get_mean_vector(model, data) \n",
    "        err.append(1)\n",
    "    except:\n",
    "        err.append(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T01:17:49.762884Z",
     "start_time": "2019-10-08T01:17:47.870940Z"
    }
   },
   "outputs": [],
   "source": [
    "vecs_all = []\n",
    "\n",
    "for i in range(0,desc.shape[0]):\n",
    "    vecs_all.append(vecs[i].tolist())\n",
    "\n",
    "comps = pd.DataFrame(vecs_all)\n",
    "comps['list_vec'] = comps.values.tolist()\n",
    "comps = comps['list_vec'].to_frame()\n",
    "comps['company'] = desc[\"company_name\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T01:18:33.448538Z",
     "start_time": "2019-10-08T01:18:33.443548Z"
    }
   },
   "outputs": [],
   "source": [
    "vecs = np.zeros((keyw.shape[0],300))\n",
    "err = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T02:46:43.379144Z",
     "start_time": "2019-10-08T02:46:43.375152Z"
    }
   },
   "outputs": [],
   "source": [
    "for i in range(0,keyw.shape[0]):\n",
    "    comp_desc = keyw.iloc[i,1]\n",
    "\n",
    "    data = []\n",
    "    # tokenize the sentence into words \n",
    "    for j in word_tokenize(comp_desc):\n",
    "        if j not in stop_words:\n",
    "            data.append(j.lower())\n",
    "    try:\n",
    "        vecs[i] = get_mean_vector(model, data)\n",
    "        err.append(1)\n",
    "    except:\n",
    "        err.append(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T01:25:23.607081Z",
     "start_time": "2019-10-08T01:25:23.489394Z"
    }
   },
   "outputs": [],
   "source": [
    "vecs_all = []\n",
    "\n",
    "for i in range(0,keyw.shape[0]):\n",
    "    vecs_all.append(vecs[i].tolist())\n",
    "\n",
    "segs = pd.DataFrame(vecs_all)\n",
    "segs['list_vec_segs'] = segs.values.tolist()\n",
    "segs = segs['list_vec_segs'].to_frame()\n",
    "segs['Industry segment'] = keyw[\"Industry segment\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Matching"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-08T01:25:53.037137Z",
     "start_time": "2019-10-08T01:25:52.903496Z"
    }
   },
   "outputs": [],
   "source": [
    "comps['key'] = 1\n",
    "segs['key'] = 1\n",
    "\n",
    "cross = pd.merge(comps, segs, on = 'key')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cross['cosine_sim'] = 0\n",
    "\n",
    "for i in range(0, cross.shape[0]):\n",
    "    vec1 = cross.iloc[i,0]\n",
    "    vec2 = cross.iloc[i,3]\n",
    "    result = 1 - spatial.distance.cosine(vec1, vec2)\n",
    "    cross.iloc[i,5] = result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fin = pd.merge(cross, cross.groupby(['company'])[\"cosine_sim\"].max(), on = [\"company\", \"cosine_sim\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fin[['company', 'Industry segment', 'cosine_sim']].to_csv(\"classify_pretrained_vectors.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "303.6px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}