tfid_Vectors-Habamungu.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          dc0.txt   dc1.txt   dc2.txt   dc3.txt   dc4.txt   dc5.txt\n",
      "worth    0.099542  0.000000  0.000000  0.000000  0.000000  0.000000\n",
      "man      0.020258  0.040516  0.010129  0.010129  0.010129  0.000000\n",
      "as       0.000000  0.000000  0.061034  0.000000  0.061034  0.000000\n",
      "of       0.099542  0.000000  0.000000  0.000000  0.000000  0.000000\n",
      "all      0.000000  0.000000  0.000000  0.000000  0.000000  0.099542\n",
      "is       0.038508  0.038508  0.000000  0.000000  0.000000  0.038508\n",
      "there    0.000000  0.000000  0.038508  0.038508  0.038508  0.000000\n",
      "are      0.000000  0.000000  0.077016  0.077016  0.077016  0.000000\n",
      "parents  0.000000  0.000000  0.038508  0.038508  0.038508  0.000000\n",
      "well     0.000000  0.000000  0.061034  0.000000  0.061034  0.000000\n",
      "not      0.000000  0.099542  0.000000  0.000000  0.000000  0.000000\n",
      "a        0.000000  0.000000  0.000000  0.000000  0.000000  0.000000\n",
      "you      0.000000  0.000000  0.000000  0.000000  0.000000  0.099542\n",
      "respect  0.000000  0.000000  0.022526  0.022526  0.022526  0.045052\n",
      "the      0.000000  0.000000  0.077016  0.077016  0.077016  0.000000\n",
      "need     0.000000  0.000000  0.000000  0.000000  0.000000  0.099542\n",
      "woman    0.061034  0.061034  0.000000  0.000000  0.000000  0.000000\n",
      "price    0.099542  0.000000  0.000000  0.000000  0.000000  0.000000\n"
     ]
    }
   ],
   "source": [
    "# include modules\n",
    "from os import listdir #here we only want to import listdir\"list directory\",\n",
    "#not the entire os module for the the sake of speed and memory \n",
    "from os.path import isfile, join #same here but what is isfile???\n",
    "import math #here we are impoting math means all formulations and stuffs.\n",
    "import re # regular expression\n",
    "from pandas import DataFrame\n",
    "\n",
    "# fetching documents\n",
    "mypath = \"C:/Users/Venant/Desktop/Habamungu\"\n",
    "\n",
    "filenames = [f for f in listdir(mypath) if isfile(join(mypath, f))]\n",
    "\n",
    "docs = []\n",
    "for file in filenames:\n",
    "    file_path = mypath + '/' + file \n",
    "    with open(file_path) as f:\n",
    "        str = ' '.join(f.readlines())\n",
    "        docs.append(str)\n",
    "\n",
    "tf_list = [] \n",
    "idf_list = []\n",
    "\n",
    "# calculations\n",
    "words_list = []\n",
    "for doc in docs:\n",
    "    words_list  += doc.split()\n",
    "words = list(set(words_list))\n",
    "#print(words_list)\n",
    "#print('\\n\\n')\n",
    "\n",
    "# calculate tf\n",
    "for word in words:\n",
    "    lst = []\n",
    "    for doc in docs:\n",
    "        word_count = sum(1 for match in re.finditer(word, doc))\n",
    "        total_word_count = len(words)\n",
    "        tf = word_count / total_word_count\n",
    "        lst.append(tf)\n",
    "    tf_list.append(lst)\n",
    "    \n",
    "df_tf = DataFrame(tf_list)\n",
    "#print('Dataframe for tf:')\n",
    "df_tf.index = words\n",
    "df_tf.columns = filenames\n",
    "#print(df_tf)\n",
    "#print('\\n\\n')\n",
    "\n",
    "# calculate idf\n",
    "\n",
    "def get_doc_count(word, docs):\n",
    "    doc_count = 0\n",
    "    for doc in docs:\n",
    "        if word in doc:\n",
    "            doc_count += 1\n",
    "    return doc_count\n",
    "\n",
    "for word in words:\n",
    "    lst = []\n",
    "    total_doc_count = len(docs)\n",
    "    doc_count = get_doc_count(word, docs)\n",
    "    for doc in docs:\n",
    "        idf = math.log(total_doc_count / doc_count)\n",
    "        #idf = math.log((total_doc_count / doc_count) , 10)\n",
    "        lst.append(idf)\n",
    "    idf_list.append(lst)\n",
    "\n",
    "df_idf = DataFrame(idf_list)\n",
    "#print('Dataframe for idf:')\n",
    "df_idf.index = words\n",
    "df_idf.columns = filenames\n",
    "#print(df_idf)\n",
    "#print('\\n\\n')\n",
    "\n",
    "# calculate tf_idf\n",
    "df_tf_idf = DataFrame(df_tf.values*df_idf.values, columns=filenames, index=words)\n",
    "#print('Dataframe for tf_idf:')\n",
    "print(df_tf_idf)\n",
    "#print('\\n\\n') #adding double space among dataframes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}