notebooks/data_processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "25c2ff35",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import datetime as dt\n",
    "import yaml\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f62a0ef5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Прочтём файл конфига с путями\n",
    "\n",
    "CONFIG_PATH = \"config.yaml\"\n",
    "with open(CONFIG_PATH, \"r\", encoding=\"utf-8\") as config_file:\n",
    "    CONFIG = yaml.load(config_file, Loader=yaml.FullLoader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "66e42bd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Загрузим необходимые датасеты\n",
    "\n",
    "df = pd.read_csv(CONFIG['datasets_folder'] + '/train_data.csv')\n",
    "post_df = pd.read_csv(CONFIG['data_folder'] + '/post_data.csv')\n",
    "user_df = pd.read_csv(CONFIG['data_folder'] + '/user_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f631ffac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>user_id</th>\n",
       "      <th>post_id</th>\n",
       "      <th>action</th>\n",
       "      <th>target</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>country</th>\n",
       "      <th>city</th>\n",
       "      <th>exp_group</th>\n",
       "      <th>os</th>\n",
       "      <th>source</th>\n",
       "      <th>text</th>\n",
       "      <th>topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021-12-02 16:33:08</td>\n",
       "      <td>125319</td>\n",
       "      <td>302</td>\n",
       "      <td>view</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>Russia</td>\n",
       "      <td>Volgograd</td>\n",
       "      <td>1</td>\n",
       "      <td>Android</td>\n",
       "      <td>organic</td>\n",
       "      <td>Ukraine steel sell-off illegal\\n\\nThe controve...</td>\n",
       "      <td>business</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021-11-09 12:43:49</td>\n",
       "      <td>37753</td>\n",
       "      <td>302</td>\n",
       "      <td>view</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>27</td>\n",
       "      <td>Ukraine</td>\n",
       "      <td>Lozova</td>\n",
       "      <td>2</td>\n",
       "      <td>Android</td>\n",
       "      <td>ads</td>\n",
       "      <td>Ukraine steel sell-off illegal\\n\\nThe controve...</td>\n",
       "      <td>business</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-11-29 16:23:52</td>\n",
       "      <td>58932</td>\n",
       "      <td>302</td>\n",
       "      <td>like</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>Russia</td>\n",
       "      <td>Moscow</td>\n",
       "      <td>1</td>\n",
       "      <td>Android</td>\n",
       "      <td>ads</td>\n",
       "      <td>Ukraine steel sell-off illegal\\n\\nThe controve...</td>\n",
       "      <td>business</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2021-11-10 07:17:43</td>\n",
       "      <td>138043</td>\n",
       "      <td>302</td>\n",
       "      <td>view</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>Russia</td>\n",
       "      <td>Pskov</td>\n",
       "      <td>0</td>\n",
       "      <td>iOS</td>\n",
       "      <td>organic</td>\n",
       "      <td>Ukraine steel sell-off illegal\\n\\nThe controve...</td>\n",
       "      <td>business</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2021-11-28 06:27:53</td>\n",
       "      <td>3635</td>\n",
       "      <td>302</td>\n",
       "      <td>view</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>Russia</td>\n",
       "      <td>Ulyanovsk</td>\n",
       "      <td>2</td>\n",
       "      <td>Android</td>\n",
       "      <td>ads</td>\n",
       "      <td>Ukraine steel sell-off illegal\\n\\nThe controve...</td>\n",
       "      <td>business</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             timestamp  user_id  post_id action  target  gender  age  country  \\\n",
       "0  2021-12-02 16:33:08   125319      302   view       1       0   22   Russia   \n",
       "1  2021-11-09 12:43:49    37753      302   view       0       1   27  Ukraine   \n",
       "2  2021-11-29 16:23:52    58932      302   like       0       0   35   Russia   \n",
       "3  2021-11-10 07:17:43   138043      302   view       0       1   22   Russia   \n",
       "4  2021-11-28 06:27:53     3635      302   view       1       1   24   Russia   \n",
       "\n",
       "        city  exp_group       os   source  \\\n",
       "0  Volgograd          1  Android  organic   \n",
       "1     Lozova          2  Android      ads   \n",
       "2     Moscow          1  Android      ads   \n",
       "3      Pskov          0      iOS  organic   \n",
       "4  Ulyanovsk          2  Android      ads   \n",
       "\n",
       "                                                text     topic  \n",
       "0  Ukraine steel sell-off illegal\\n\\nThe controve...  business  \n",
       "1  Ukraine steel sell-off illegal\\n\\nThe controve...  business  \n",
       "2  Ukraine steel sell-off illegal\\n\\nThe controve...  business  \n",
       "3  Ukraine steel sell-off illegal\\n\\nThe controve...  business  \n",
       "4  Ukraine steel sell-off illegal\\n\\nThe controve...  business  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4cd81baa",
   "metadata": {},
   "source": [
    "##### Обработаем признаки"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0bd78a14",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Временные признаки - выделим из времени совершённого действия месяц, число, день недели и час\n",
    "\n",
    "df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"])\n",
    "\n",
    "df['month'] = df[\"timestamp\"].dt.month.astype(int)\n",
    "df['hour'] = df[\"timestamp\"].dt.hour.astype(int)\n",
    "df['day'] = df[\"timestamp\"].dt.day.astype(int)\n",
    "df['weekday'] = df[\"timestamp\"].dt.weekday.astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4f1622f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Приведем некоторые категориальные признаки к бинарному виду\n",
    "\n",
    "df['os'] = df['os'].map({'Android': 0, 'iOS': 1})\n",
    "df['source'] = df['source'].map({'ads': 0, 'organic': 1})\n",
    "\n",
    "user_df['os'] = user_df['os'].map({'Android': 0, 'iOS': 1})\n",
    "user_df['source'] = user_df['source'].map({'ads': 0, 'organic': 1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b041f90f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Уберем колонку с текстом новости, т.к. обработка текста будет вестись через post_data.csv\n",
    "\n",
    "df = df.drop(['text'], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eaf5fb2a",
   "metadata": {},
   "source": [
    "##### Выделим текстовые признаки"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "dae29cb9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>post_id</th>\n",
       "      <th>text</th>\n",
       "      <th>topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>UK economy facing major risks\\n\\nThe UK manufa...</td>\n",
       "      <td>business</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Aids and climate top Davos agenda\\n\\nClimate c...</td>\n",
       "      <td>business</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Asian quake hits European shares\\n\\nShares in ...</td>\n",
       "      <td>business</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>India power shares jump on debut\\n\\nShares in ...</td>\n",
       "      <td>business</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Lacroix label bought by US firm\\n\\nLuxury good...</td>\n",
       "      <td>business</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   post_id                                               text     topic\n",
       "0        1  UK economy facing major risks\\n\\nThe UK manufa...  business\n",
       "1        2  Aids and climate top Davos agenda\\n\\nClimate c...  business\n",
       "2        3  Asian quake hits European shares\\n\\nShares in ...  business\n",
       "3        4  India power shares jump on debut\\n\\nShares in ...  business\n",
       "4        5  Lacroix label bought by US firm\\n\\nLuxury good...  business"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "post_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "24619fd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Уберём символ переноса строки\n",
    "\n",
    "post_df['text'] = post_df['text'].str.replace('\\n',' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "05a98ac3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Найдем tf-idf матрицу нашего корпуса текстов и выделим признаки\n",
    "\n",
    "tfidf = TfidfVectorizer(stop_words='english', ngram_range=(1, 3))\n",
    "tfidf_df_ = tfidf.fit_transform(post_df['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "929bcc78",
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_df = []\n",
    "\n",
    "for row in tfidf_df_:\n",
    "    tfidf_df.append([row.sum(), row.mean(), row.max()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "758517d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_df = pd.DataFrame(tfidf_df, index=post_df['post_id'], columns=['tfidf_sum', 'tfidf_mean', 'tfidf_max']).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2a9beccd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>post_id</th>\n",
       "      <th>tfidf_sum</th>\n",
       "      <th>tfidf_mean</th>\n",
       "      <th>tfidf_max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>20.580054</td>\n",
       "      <td>0.000015</td>\n",
       "      <td>0.249396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>25.240271</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>0.146967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>28.351763</td>\n",
       "      <td>0.000021</td>\n",
       "      <td>0.143002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>16.025785</td>\n",
       "      <td>0.000012</td>\n",
       "      <td>0.287929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>14.818161</td>\n",
       "      <td>0.000011</td>\n",
       "      <td>0.231884</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   post_id  tfidf_sum  tfidf_mean  tfidf_max\n",
       "0        1  20.580054    0.000015   0.249396\n",
       "1        2  25.240271    0.000019   0.146967\n",
       "2        3  28.351763    0.000021   0.143002\n",
       "3        4  16.025785    0.000012   0.287929\n",
       "4        5  14.818161    0.000011   0.231884"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0bc255ed",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Получим финальный датасет, на котором будем обучать модель и сохраним вместе с tf-idf признаками\n",
    "\n",
    "df = df.merge(tfidf_df, on='post_id')\n",
    "df = df.sort_values(by='timestamp')\n",
    "\n",
    "df.to_csv(CONFIG['datasets_folder'] + '/processed_df.csv', index=False)\n",
    "tfidf_df.to_csv(CONFIG['data_folder'] + '/tfidf_df.csv', index=False)\n",
    "post_df.to_csv(CONFIG['data_folder'] + '/post_data.csv', index=False)\n",
    "user_df.to_csv(CONFIG['data_folder'] + '/user_data.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7807933",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}