ml_07_Feature_Engineering-HW.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "constitutional-helping",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.preprocessing import RobustScaler\n",
    "from sklearn.feature_selection import SelectKBest, SelectPercentile\n",
    "from sklearn.feature_selection import mutual_info_classif, chi2\n",
    "from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor\n",
    "from sklearn.metrics import roc_auc_score, mean_squared_error\n",
    "\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from imblearn.under_sampling import CondensedNearestNeighbour\n",
    "\n",
    "import category_encoders as ce\n",
    "\n",
    "plt.style.use('seaborn-colorblind')\n",
    "%matplotlib inline\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "green-thermal",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(32561, 14)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>workclass</th>\n",
       "      <th>fnlwgt</th>\n",
       "      <th>education</th>\n",
       "      <th>education-num</th>\n",
       "      <th>marital-status</th>\n",
       "      <th>occupation</th>\n",
       "      <th>relationship</th>\n",
       "      <th>race</th>\n",
       "      <th>sex</th>\n",
       "      <th>capital-gain</th>\n",
       "      <th>capital-loss</th>\n",
       "      <th>hours-per-week</th>\n",
       "      <th>native-country</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>39</td>\n",
       "      <td>State-gov</td>\n",
       "      <td>77516</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Adm-clerical</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>2174</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>50</td>\n",
       "      <td>Self-emp-not-inc</td>\n",
       "      <td>83311</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Exec-managerial</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>38</td>\n",
       "      <td>Private</td>\n",
       "      <td>215646</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>Handlers-cleaners</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>53</td>\n",
       "      <td>Private</td>\n",
       "      <td>234721</td>\n",
       "      <td>11th</td>\n",
       "      <td>7</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Handlers-cleaners</td>\n",
       "      <td>Husband</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>Private</td>\n",
       "      <td>338409</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Prof-specialty</td>\n",
       "      <td>Wife</td>\n",
       "      <td>Black</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>Cuba</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>37</td>\n",
       "      <td>Private</td>\n",
       "      <td>284582</td>\n",
       "      <td>Masters</td>\n",
       "      <td>14</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Exec-managerial</td>\n",
       "      <td>Wife</td>\n",
       "      <td>White</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>49</td>\n",
       "      <td>Private</td>\n",
       "      <td>160187</td>\n",
       "      <td>9th</td>\n",
       "      <td>5</td>\n",
       "      <td>Married-spouse-absent</td>\n",
       "      <td>Other-service</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>Black</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>Jamaica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>52</td>\n",
       "      <td>Self-emp-not-inc</td>\n",
       "      <td>209642</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Exec-managerial</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>45</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age          workclass  fnlwgt   education  education-num  \\\n",
       "0   39          State-gov   77516   Bachelors             13   \n",
       "1   50   Self-emp-not-inc   83311   Bachelors             13   \n",
       "2   38            Private  215646     HS-grad              9   \n",
       "3   53            Private  234721        11th              7   \n",
       "4   28            Private  338409   Bachelors             13   \n",
       "5   37            Private  284582     Masters             14   \n",
       "6   49            Private  160187         9th              5   \n",
       "7   52   Self-emp-not-inc  209642     HS-grad              9   \n",
       "\n",
       "           marital-status          occupation    relationship    race  \\\n",
       "0           Never-married        Adm-clerical   Not-in-family   White   \n",
       "1      Married-civ-spouse     Exec-managerial         Husband   White   \n",
       "2                Divorced   Handlers-cleaners   Not-in-family   White   \n",
       "3      Married-civ-spouse   Handlers-cleaners         Husband   Black   \n",
       "4      Married-civ-spouse      Prof-specialty            Wife   Black   \n",
       "5      Married-civ-spouse     Exec-managerial            Wife   White   \n",
       "6   Married-spouse-absent       Other-service   Not-in-family   Black   \n",
       "7      Married-civ-spouse     Exec-managerial         Husband   White   \n",
       "\n",
       "       sex  capital-gain  capital-loss  hours-per-week  native-country  \n",
       "0     Male          2174             0              40   United-States  \n",
       "1     Male             0             0              13   United-States  \n",
       "2     Male             0             0              40   United-States  \n",
       "3     Male             0             0              40   United-States  \n",
       "4   Female             0             0              40            Cuba  \n",
       "5   Female             0             0              40   United-States  \n",
       "6   Female             0             0              16         Jamaica  \n",
       "7     Male             0             0              45   United-States  "
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "use_cols = [\n",
    "    'age',\n",
    "    'workclass',\n",
    "    'fnlwgt',\n",
    "    'education',\n",
    "    'education-num',\n",
    "    'marital-status',\n",
    "    'occupation',\n",
    "    'relationship',\n",
    "    'race',\n",
    "    'sex',\n",
    "    'capital-gain',\n",
    "    'capital-loss',\n",
    "    'hours-per-week',\n",
    "    'native-country'\n",
    "]\n",
    "\n",
    "data = pd.read_csv('datasets/adult.data', index_col=False,\n",
    "                   names=[\n",
    "                            'age',\n",
    "                            'workclass',\n",
    "                            'fnlwgt',\n",
    "                            'education',\n",
    "                            'education-num',\n",
    "                            'marital-status',\n",
    "                            'occupation',\n",
    "                            'relationship',\n",
    "                            'race',\n",
    "                            'sex',\n",
    "                            'capital-gain',\n",
    "                            'capital-loss',\n",
    "                            'hours-per-week',\n",
    "                            'native-country'])\n",
    "print(data.shape)\n",
    "data.head(8)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "continent-attraction",
   "metadata": {},
   "source": [
    "## Пропущенные значения"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sunrise-result",
   "metadata": {},
   "source": [
    "Проверка пропущенных значений"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "coordinate-missouri",
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_missing(data, output_path=None):\n",
    "    result = pd.concat([data.isnull().sum(), data.isnull().mean()], axis=1)\n",
    "    result = result.rename(index=str, columns={0: 'total missing', 1: 'proportion'})\n",
    "    if output_path:\n",
    "        result.to_csv(output_path + 'missing.csv')\n",
    "        print(output_path + 'missing.csv')\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "insured-specification",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "datasets/missing.csv\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>total missing</th>\n",
       "      <th>proportion</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>age</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>workclass</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fnlwgt</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>education</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>education-num</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>marital-status</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>occupation</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>relationship</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>race</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sex</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>capital-gain</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>capital-loss</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hours-per-week</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>native-country</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                total missing  proportion\n",
       "age                         0         0.0\n",
       "workclass                   0         0.0\n",
       "fnlwgt                      0         0.0\n",
       "education                   0         0.0\n",
       "education-num               0         0.0\n",
       "marital-status              0         0.0\n",
       "occupation                  0         0.0\n",
       "relationship                0         0.0\n",
       "race                        0         0.0\n",
       "sex                         0         0.0\n",
       "capital-gain                0         0.0\n",
       "capital-loss                0         0.0\n",
       "hours-per-week              0         0.0\n",
       "native-country              0         0.0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_missing(data, 'datasets/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "native-dispatch",
   "metadata": {},
   "outputs": [],
   "source": [
    "def drop_missing(data, axis=0):\n",
    "    data_copy = data.copy()\n",
    "    data_copy = data_copy.dropna(axis=axis, inplace=False)\n",
    "    return data_copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "minor-behavior",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32561, 14)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data2 = drop_missing(data)\n",
    "data2.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "logical-dimension",
   "metadata": {},
   "source": [
    "Добавление переменной оценки пропущенных значений"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "driving-criminal",
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_var_denote_NA(data, NA_col=[]):\n",
    "    data_copy = data.copy()\n",
    "    for i in NA_col:\n",
    "        if data_copy[i].isnull().sum() > 0:\n",
    "            data_copy[i+'_is_NA'] = np.where(data_copy[i].isnull(), 1, 0)\n",
    "        else:\n",
    "             warn('Нет пропущенных значений' % i)\n",
    "    return data_copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "common-aerospace",
   "metadata": {},
   "outputs": [],
   "source": [
    "# data3 = add_var_denote_NA(data, NA_col=['age'])\n",
    "# print(data3.Age_is_NA.value_counts())\n",
    "# data3.head(8)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "national-salvation",
   "metadata": {},
   "source": [
    "Заполнение пропусков выборочным значением"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "simple-frederick",
   "metadata": {},
   "outputs": [],
   "source": [
    "def impute_NA_with_arbitrary(data, impute_value, NA_col=[]):\n",
    "    data_copy = data.copy()\n",
    "    for i in NA_col:\n",
    "        if data_copy[i].isnull().sum() > 0:\n",
    "            data_copy[i + '_' + str(impute_value)] = data_copy[i].fillna(impute_value)\n",
    "        else:\n",
    "            warn('Нет пропущенных знаечний' % i)\n",
    "    return data_copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "functional-beatles",
   "metadata": {},
   "outputs": [],
   "source": [
    "# data4 = impute_NA_with_arbitrary(data, -999, ['age'])\n",
    "# data4.head(8)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "funky-advocacy",
   "metadata": {},
   "source": [
    "Заполнение пропущенных значений средним/медианой/модой"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "czech-mortality",
   "metadata": {},
   "outputs": [],
   "source": [
    "def impute_with_avg(data, strategy='mean', NA_col=[]):\n",
    "    data_copy = data.copy()\n",
    "    for i in NA_col:\n",
    "        if data_copy[i].isnull().sum() > 0:\n",
    "            if strategy == 'mean':\n",
    "                data_copy[i + '_impute_mean'] = data_copy[i].fillna(data[i].mean())\n",
    "            elif strategy == 'median':\n",
    "                data_copy[i + '_impute_median'] = data_copy[i].fillna(data[i].median())\n",
    "            elif strategy == 'mode':\n",
    "                data_copy[i + '_impute_mode'] = data_copy[i].fillna(data[i].mode())\n",
    "        else:\n",
    "            warn('Нет пропущенных данных' % i)\n",
    "    return data_copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "boolean-serbia",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(data.Age.median())\n",
    "# data5 = impute_with_avg(data, 'median', ['age'])\n",
    "# data5.head(8)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "parental-footage",
   "metadata": {},
   "source": [
    "Заполнение пропусков из хвоста распределения"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "outdoor-happiness",
   "metadata": {},
   "outputs": [],
   "source": [
    "def impute_NA_with_end_of_distribution(data, NA_col=[]):\n",
    "    data_copy = data.copy()\n",
    "    for i in NA_col:\n",
    "        if data_copy[i].isnull().sum() > 0:\n",
    "            data_copy[i + '_impute_end_of_distri'] = data_copy[i].fillna(data[i].mean() + 3 * data[i].std())\n",
    "        else:\n",
    "            warn('Нет пропущенных данных' % i)\n",
    "    return data_copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "spatial-philosophy",
   "metadata": {},
   "outputs": [],
   "source": [
    "# data6 = impute_NA_with_end_of_distribution(data, ['age'])\n",
    "# data6.head(8)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "pacific-therapist",
   "metadata": {},
   "source": [
    "Заполнение пропусков случайными значениями"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "least-disorder",
   "metadata": {},
   "outputs": [],
   "source": [
    "def imput_NA_with_random(data, NA_col=[], random_state=0):\n",
    "    data_copy = data.copy()\n",
    "    for i in NA_col:\n",
    "        if data_copy[i].isnull().sum() > 0:\n",
    "            data_copy[i + '_random'] = data_copy[i]\n",
    "            random_sample = data_copy[i].dropna().sample(data_copy[i].isnull().sum(), random_state=random_state)\n",
    "            random_sample.index = data_copy[data_copy[i].isnull()].index\n",
    "            data_copy.loc[data_copy[i].isnull(), str(i) + '_random'] = random_sample\n",
    "        else:\n",
    "            warn('Нет пропущенных данных' % i)\n",
    "    return data_copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "sufficient-workplace",
   "metadata": {},
   "outputs": [],
   "source": [
    "# data7 = imput_NA_with_random(data, ['age'])\n",
    "# data7.head(8)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "transparent-portable",
   "metadata": {},
   "source": [
    "# Выбросы"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "signed-plenty",
   "metadata": {},
   "source": [
    "Детекция с помощью выборочных значений"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "gentle-bowling",
   "metadata": {},
   "outputs": [],
   "source": [
    "def outlier_detect_arbitrary(data, col, uppper_fence, lower_fence):\n",
    "    para = (uppper_fence, lower_fence)\n",
    "    tmp = pd.concat([data[col] > uppper_fence, data[col] < lower_fence], axis=1)\n",
    "    outlier_index = tmp.any(axis=1)\n",
    "    print('Количество выбросов в данных:', outlier_index.value_counts()[1])\n",
    "    print('Доля выбросов', outlier_index.value_counts()[1]/len(outlier_index))\n",
    "    return outlier_index, para"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "herbal-permission",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "90"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.age.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "tested-winner",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Количество выбросов в данных: 43\n",
      "Доля выбросов 0.0013205982617241485\n",
      "Верхняя граница: 89 \n",
      "Нижняя граница: 5\n"
     ]
    }
   ],
   "source": [
    "index, para = outlier_detect_arbitrary(data, 'age', 89, 5)\n",
    "print('Верхняя граница:', para[0], '\\nНижняя граница:', para[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "unnecessary-hunger",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "222      90\n",
       "14159    90\n",
       "15356    90\n",
       "15892    90\n",
       "18277    90\n",
       "18413    90\n",
       "18725    90\n",
       "18832    90\n",
       "19212    90\n",
       "12975    90\n",
       "19489    90\n",
       "20610    90\n",
       "22220    90\n",
       "24043    90\n",
       "24238    90\n",
       "25303    90\n",
       "28463    90\n",
       "31030    90\n",
       "31696    90\n",
       "19747    90\n",
       "32277    90\n",
       "12451    90\n",
       "11731    90\n",
       "1040     90\n",
       "1935     90\n",
       "2303     90\n",
       "2891     90\n",
       "4070     90\n",
       "4109     90\n",
       "5104     90\n",
       "5272     90\n",
       "11996    90\n",
       "5370     90\n",
       "6232     90\n",
       "6624     90\n",
       "8806     90\n",
       "8963     90\n",
       "8973     90\n",
       "10210    90\n",
       "10545    90\n",
       "11512    90\n",
       "5406     90\n",
       "32367    90\n",
       "Name: age, dtype: int64"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.loc[index, 'age'].sort_values()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "industrial-frame",
   "metadata": {},
   "source": [
    "Интерквартильное расстояние"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "familiar-physiology",
   "metadata": {},
   "outputs": [],
   "source": [
    "def outlier_detect_IQR(data, col, threshold=3):\n",
    "    IQR = data[col].quantile(0.75) - data[col].quantile(0.25)\n",
    "    Lower_fence = data[col].quantile(0.25) - (IQR * threshold)\n",
    "    Upper_fence = data[col].quantile(0.75) + (IQR * threshold)\n",
    "    para = (Upper_fence, Lower_fence)\n",
    "    tmp = pd.concat([data[col] > Upper_fence, data[col] < Lower_fence], axis=1)\n",
    "    outlier_index = tmp.any(axis=1)\n",
    "    print('Количество выбросов в данных:', outlier_index.value_counts()[1])\n",
    "    print('Доля выбросов', outlier_index.value_counts()[1]/len(outlier_index))\n",
    "    return outlier_index, para"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "unauthorized-petite",
   "metadata": {},
   "outputs": [],
   "source": [
    "# index, para = outlier_detect_IQR(data, 'age', 3)\n",
    "# print('Верхняя граница:', para[0], '\\nНижняя граница:', para[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "buried-drive",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "222      90\n",
       "14159    90\n",
       "15356    90\n",
       "15892    90\n",
       "18277    90\n",
       "18413    90\n",
       "18725    90\n",
       "18832    90\n",
       "19212    90\n",
       "12975    90\n",
       "19489    90\n",
       "20610    90\n",
       "22220    90\n",
       "24043    90\n",
       "24238    90\n",
       "25303    90\n",
       "28463    90\n",
       "31030    90\n",
       "31696    90\n",
       "19747    90\n",
       "32277    90\n",
       "12451    90\n",
       "11731    90\n",
       "1040     90\n",
       "1935     90\n",
       "2303     90\n",
       "2891     90\n",
       "4070     90\n",
       "4109     90\n",
       "5104     90\n",
       "5272     90\n",
       "11996    90\n",
       "5370     90\n",
       "6232     90\n",
       "6624     90\n",
       "8806     90\n",
       "8963     90\n",
       "8973     90\n",
       "10210    90\n",
       "10545    90\n",
       "11512    90\n",
       "5406     90\n",
       "32367    90\n",
       "Name: age, dtype: int64"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.loc[index, 'age'].sort_values()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "nervous-jewelry",
   "metadata": {},
   "source": [
    "Среднее-среднеквадратичное отклонение"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "surprising-barbados",
   "metadata": {},
   "outputs": [],
   "source": [
    "def outlier_detect_mean_std(data, col, threshold=3):\n",
    "    Upper_fence = data[col].mean() + threshold * data[col].std()\n",
    "    Lower_fence = data[col].mean() - threshold * data[col].std()\n",
    "    para = (Upper_fence, Lower_fence)\n",
    "    tmp = pd.concat([data[col] > Upper_fence, data[col] < Lower_fence], axis=1)\n",
    "    outlier_index = tmp.any(axis=1)\n",
    "    print('Количество выбросов в данных:', outlier_index.value_counts()[1])\n",
    "    print('Доля выбросов', outlier_index.value_counts()[1]/len(outlier_index))\n",
    "    return outlier_index, para"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "animal-victim",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Количество выбросов в данных: 121\n",
      "Доля выбросов 0.00371610208531679\n",
      "Верхняя граница: 79.5029444160648 \n",
      "Нижняя граница: -2.339650905423241\n"
     ]
    }
   ],
   "source": [
    "index, para = outlier_detect_mean_std(data, 'age', 3)\n",
    "print('Верхняя граница:', para[0], '\\nНижняя граница:', para[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "attended-physics",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20953    80\n",
       "20482    80\n",
       "20249    80\n",
       "6439     80\n",
       "19689    80\n",
       "         ..\n",
       "20610    90\n",
       "6624     90\n",
       "15356    90\n",
       "19212    90\n",
       "222      90\n",
       "Name: age, Length: 121, dtype: int64"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.loc[index, 'age'].sort_values()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "collectible-madison",
   "metadata": {},
   "source": [
    "Медианы абсолютного отклонения (MAD)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "partial-holiday",
   "metadata": {},
   "outputs": [],
   "source": [
    "def outlier_detect_mad(data, col, threshold=3.5):\n",
    "    median = data[col].median()\n",
    "    median_absolute_deviation = np.median([np.abs(y - median) for y in data[col]])\n",
    "    modified_z_scores = pd.Series([0.6745 * (y -median) / median_absolute_deviation for y in data[col]])\n",
    "    outlier_index = np.abs(modified_z_scores) > threshold\n",
    "    print('Количество выбросов в данных:', outlier_index.value_counts()[1])\n",
    "    print('Доля выбросов', outlier_index.value_counts()[1]/len(outlier_index))\n",
    "    return outlier_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "colonial-output",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Количество выбросов в данных: 43\n",
      "Доля выбросов 0.0013205982617241485\n"
     ]
    }
   ],
   "source": [
    "index = outlier_detect_mad(data, 'age', 3.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "hungarian-nickname",
   "metadata": {},
   "source": [
    "Замена выброса выборочным значением"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "small-harvey",
   "metadata": {},
   "outputs": [],
   "source": [
    "def impute_outlier_with_arbitrary(data, outlier_index, value, col=[]):\n",
    "    data_copy = data.copy()\n",
    "    for i in col:\n",
    "        data_copy.loc[outlier_index, i] = value\n",
    "    return data_copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "resident-contributor",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>workclass</th>\n",
       "      <th>fnlwgt</th>\n",
       "      <th>education</th>\n",
       "      <th>education-num</th>\n",
       "      <th>marital-status</th>\n",
       "      <th>occupation</th>\n",
       "      <th>relationship</th>\n",
       "      <th>race</th>\n",
       "      <th>sex</th>\n",
       "      <th>capital-gain</th>\n",
       "      <th>capital-loss</th>\n",
       "      <th>hours-per-week</th>\n",
       "      <th>native-country</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>56</td>\n",
       "      <td>Local-gov</td>\n",
       "      <td>216851</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Tech-support</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>19</td>\n",
       "      <td>Private</td>\n",
       "      <td>168294</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Craft-repair</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>54</td>\n",
       "      <td>?</td>\n",
       "      <td>180211</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>10</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>?</td>\n",
       "      <td>Husband</td>\n",
       "      <td>Asian-Pac-Islander</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>60</td>\n",
       "      <td>South</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>39</td>\n",
       "      <td>Private</td>\n",
       "      <td>367260</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>Exec-managerial</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>80</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>49</td>\n",
       "      <td>Private</td>\n",
       "      <td>193366</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Craft-repair</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>23</td>\n",
       "      <td>Local-gov</td>\n",
       "      <td>190709</td>\n",
       "      <td>Assoc-acdm</td>\n",
       "      <td>12</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Protective-serv</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>20</td>\n",
       "      <td>Private</td>\n",
       "      <td>266015</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>10</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Sales</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>44</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>45</td>\n",
       "      <td>Private</td>\n",
       "      <td>386940</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>Exec-managerial</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>1408</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>30</td>\n",
       "      <td>Federal-gov</td>\n",
       "      <td>59951</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>10</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Adm-clerical</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>22</td>\n",
       "      <td>State-gov</td>\n",
       "      <td>311512</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>10</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Other-service</td>\n",
       "      <td>Husband</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    age     workclass  fnlwgt      education  education-num  \\\n",
       "25   56     Local-gov  216851      Bachelors             13   \n",
       "26   19       Private  168294        HS-grad              9   \n",
       "27   54             ?  180211   Some-college             10   \n",
       "28   39       Private  367260        HS-grad              9   \n",
       "29   49       Private  193366        HS-grad              9   \n",
       "30   23     Local-gov  190709     Assoc-acdm             12   \n",
       "31   20       Private  266015   Some-college             10   \n",
       "32   45       Private  386940      Bachelors             13   \n",
       "33   30   Federal-gov   59951   Some-college             10   \n",
       "34   22     State-gov  311512   Some-college             10   \n",
       "\n",
       "         marital-status        occupation    relationship  \\\n",
       "25   Married-civ-spouse      Tech-support         Husband   \n",
       "26        Never-married      Craft-repair       Own-child   \n",
       "27   Married-civ-spouse                 ?         Husband   \n",
       "28             Divorced   Exec-managerial   Not-in-family   \n",
       "29   Married-civ-spouse      Craft-repair         Husband   \n",
       "30        Never-married   Protective-serv   Not-in-family   \n",
       "31        Never-married             Sales       Own-child   \n",
       "32             Divorced   Exec-managerial       Own-child   \n",
       "33   Married-civ-spouse      Adm-clerical       Own-child   \n",
       "34   Married-civ-spouse     Other-service         Husband   \n",
       "\n",
       "                   race    sex  capital-gain  capital-loss  hours-per-week  \\\n",
       "25                White   Male             0             0              40   \n",
       "26                White   Male             0             0              40   \n",
       "27   Asian-Pac-Islander   Male             0             0              60   \n",
       "28                White   Male             0             0              80   \n",
       "29                White   Male             0             0              40   \n",
       "30                White   Male             0             0              52   \n",
       "31                Black   Male             0             0              44   \n",
       "32                White   Male             0          1408              40   \n",
       "33                White   Male             0             0              40   \n",
       "34                Black   Male             0             0              15   \n",
       "\n",
       "    native-country  \n",
       "25   United-States  \n",
       "26   United-States  \n",
       "27           South  \n",
       "28   United-States  \n",
       "29   United-States  \n",
       "30   United-States  \n",
       "31   United-States  \n",
       "32   United-States  \n",
       "33   United-States  \n",
       "34   United-States  "
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data2 = impute_outlier_with_arbitrary(data, index, -999, ['age'])\n",
    "data2[25:35]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "casual-ribbon",
   "metadata": {},
   "source": [
    "Винзоризация"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "addressed-concern",
   "metadata": {},
   "source": [
    "{92, 19, 101, 58, 1053, 91, 26, 78, 10, 13, −40, 101, 86, 85, 15, 89, 89, 28, −5, 41}  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "understanding-physiology",
   "metadata": {},
   "source": [
    "{92, 19, 101, 58, 101, 91, 26, 78, 10, 13, −5, 101, 86, 85, 15, 89, 89, 28, −5, 41} "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "backed-customs",
   "metadata": {},
   "outputs": [],
   "source": [
    "def winsorization(data, col, para, strategy='both'):\n",
    "    data_copy = data.copy()\n",
    "    if strategy == 'both':\n",
    "        data_copy.loc[data_copy[col] > para[0], col] = para[0]\n",
    "        data_copy.loc[data_copy[col] < para[1], col] = para[1]\n",
    "    elif strategy == 'top':\n",
    "        data_copy.loc[data_copy[col] > para[0], col] = para[0]\n",
    "    elif strategy == 'bottom':\n",
    "        data_copy.loc[data_copy[col] > para[1], col] = para[1]\n",
    "    return data_copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "placed-indonesia",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>workclass</th>\n",
       "      <th>fnlwgt</th>\n",
       "      <th>education</th>\n",
       "      <th>education-num</th>\n",
       "      <th>marital-status</th>\n",
       "      <th>occupation</th>\n",
       "      <th>relationship</th>\n",
       "      <th>race</th>\n",
       "      <th>sex</th>\n",
       "      <th>capital-gain</th>\n",
       "      <th>capital-loss</th>\n",
       "      <th>hours-per-week</th>\n",
       "      <th>native-country</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>56.0</td>\n",
       "      <td>Local-gov</td>\n",
       "      <td>216851</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Tech-support</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>19.0</td>\n",
       "      <td>Private</td>\n",
       "      <td>168294</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Craft-repair</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>54.0</td>\n",
       "      <td>?</td>\n",
       "      <td>180211</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>10</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>?</td>\n",
       "      <td>Husband</td>\n",
       "      <td>Asian-Pac-Islander</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>60</td>\n",
       "      <td>South</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>39.0</td>\n",
       "      <td>Private</td>\n",
       "      <td>367260</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>Exec-managerial</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>80</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>49.0</td>\n",
       "      <td>Private</td>\n",
       "      <td>193366</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Craft-repair</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>23.0</td>\n",
       "      <td>Local-gov</td>\n",
       "      <td>190709</td>\n",
       "      <td>Assoc-acdm</td>\n",
       "      <td>12</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Protective-serv</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>20.0</td>\n",
       "      <td>Private</td>\n",
       "      <td>266015</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>10</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Sales</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>44</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>45.0</td>\n",
       "      <td>Private</td>\n",
       "      <td>386940</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>Exec-managerial</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>1408</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>30.0</td>\n",
       "      <td>Federal-gov</td>\n",
       "      <td>59951</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>10</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Adm-clerical</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>22.0</td>\n",
       "      <td>State-gov</td>\n",
       "      <td>311512</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>10</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Other-service</td>\n",
       "      <td>Husband</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     age     workclass  fnlwgt      education  education-num  \\\n",
       "25  56.0     Local-gov  216851      Bachelors             13   \n",
       "26  19.0       Private  168294        HS-grad              9   \n",
       "27  54.0             ?  180211   Some-college             10   \n",
       "28  39.0       Private  367260        HS-grad              9   \n",
       "29  49.0       Private  193366        HS-grad              9   \n",
       "30  23.0     Local-gov  190709     Assoc-acdm             12   \n",
       "31  20.0       Private  266015   Some-college             10   \n",
       "32  45.0       Private  386940      Bachelors             13   \n",
       "33  30.0   Federal-gov   59951   Some-college             10   \n",
       "34  22.0     State-gov  311512   Some-college             10   \n",
       "\n",
       "         marital-status        occupation    relationship  \\\n",
       "25   Married-civ-spouse      Tech-support         Husband   \n",
       "26        Never-married      Craft-repair       Own-child   \n",
       "27   Married-civ-spouse                 ?         Husband   \n",
       "28             Divorced   Exec-managerial   Not-in-family   \n",
       "29   Married-civ-spouse      Craft-repair         Husband   \n",
       "30        Never-married   Protective-serv   Not-in-family   \n",
       "31        Never-married             Sales       Own-child   \n",
       "32             Divorced   Exec-managerial       Own-child   \n",
       "33   Married-civ-spouse      Adm-clerical       Own-child   \n",
       "34   Married-civ-spouse     Other-service         Husband   \n",
       "\n",
       "                   race    sex  capital-gain  capital-loss  hours-per-week  \\\n",
       "25                White   Male             0             0              40   \n",
       "26                White   Male             0             0              40   \n",
       "27   Asian-Pac-Islander   Male             0             0              60   \n",
       "28                White   Male             0             0              80   \n",
       "29                White   Male             0             0              40   \n",
       "30                White   Male             0             0              52   \n",
       "31                Black   Male             0             0              44   \n",
       "32                White   Male             0          1408              40   \n",
       "33                White   Male             0             0              40   \n",
       "34                Black   Male             0             0              15   \n",
       "\n",
       "    native-country  \n",
       "25   United-States  \n",
       "26   United-States  \n",
       "27           South  \n",
       "28   United-States  \n",
       "29   United-States  \n",
       "30   United-States  \n",
       "31   United-States  \n",
       "32   United-States  \n",
       "33   United-States  \n",
       "34   United-States  "
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data3 = winsorization(data, 'age', para, 'both')\n",
    "data3[25:35]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "geological-kentucky",
   "metadata": {},
   "source": [
    "Удаление выбросов"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "western-median",
   "metadata": {},
   "outputs": [],
   "source": [
    "def drop_outlier(data, outlier_index):\n",
    "    data_copy = data[-outlier_index]\n",
    "    return data_copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "cordless-rochester",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "88\n",
      "17\n"
     ]
    }
   ],
   "source": [
    "data4 = drop_outlier(data, index)\n",
    "print(data4.age.max())\n",
    "print(data4.age.min())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "wound-scheduling",
   "metadata": {},
   "source": [
    "Замена выбросов средним/медианой/модой"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "resident-peter",
   "metadata": {},
   "outputs": [],
   "source": [
    "def impute_outlier_with_avg(data, col, outlier_index, strategy='mean'):\n",
    "    data_copy = data.copy()\n",
    "    if strategy == 'mean':\n",
    "        data_copy.loc[outlier_index, col] = data_copy[col].mean()\n",
    "    elif strategy == 'median':\n",
    "        data_copy.loc[outlier_index, col] = data_copy[col].median()\n",
    "    elif strategy == 'mode':\n",
    "        data_copy.loc[outlier_index, col] = data_copy[col].mode()[0]\n",
    "    return data_copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "minus-kernel",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>workclass</th>\n",
       "      <th>fnlwgt</th>\n",
       "      <th>education</th>\n",
       "      <th>education-num</th>\n",
       "      <th>marital-status</th>\n",
       "      <th>occupation</th>\n",
       "      <th>relationship</th>\n",
       "      <th>race</th>\n",
       "      <th>sex</th>\n",
       "      <th>capital-gain</th>\n",
       "      <th>capital-loss</th>\n",
       "      <th>hours-per-week</th>\n",
       "      <th>native-country</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>56.0</td>\n",
       "      <td>Local-gov</td>\n",
       "      <td>216851</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Tech-support</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>19.0</td>\n",
       "      <td>Private</td>\n",
       "      <td>168294</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Craft-repair</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>54.0</td>\n",
       "      <td>?</td>\n",
       "      <td>180211</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>10</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>?</td>\n",
       "      <td>Husband</td>\n",
       "      <td>Asian-Pac-Islander</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>60</td>\n",
       "      <td>South</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>39.0</td>\n",
       "      <td>Private</td>\n",
       "      <td>367260</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>Exec-managerial</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>80</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>49.0</td>\n",
       "      <td>Private</td>\n",
       "      <td>193366</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Craft-repair</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>23.0</td>\n",
       "      <td>Local-gov</td>\n",
       "      <td>190709</td>\n",
       "      <td>Assoc-acdm</td>\n",
       "      <td>12</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Protective-serv</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>20.0</td>\n",
       "      <td>Private</td>\n",
       "      <td>266015</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>10</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Sales</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>44</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>45.0</td>\n",
       "      <td>Private</td>\n",
       "      <td>386940</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>Exec-managerial</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>1408</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>30.0</td>\n",
       "      <td>Federal-gov</td>\n",
       "      <td>59951</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>10</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Adm-clerical</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>22.0</td>\n",
       "      <td>State-gov</td>\n",
       "      <td>311512</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>10</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Other-service</td>\n",
       "      <td>Husband</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     age     workclass  fnlwgt      education  education-num  \\\n",
       "25  56.0     Local-gov  216851      Bachelors             13   \n",
       "26  19.0       Private  168294        HS-grad              9   \n",
       "27  54.0             ?  180211   Some-college             10   \n",
       "28  39.0       Private  367260        HS-grad              9   \n",
       "29  49.0       Private  193366        HS-grad              9   \n",
       "30  23.0     Local-gov  190709     Assoc-acdm             12   \n",
       "31  20.0       Private  266015   Some-college             10   \n",
       "32  45.0       Private  386940      Bachelors             13   \n",
       "33  30.0   Federal-gov   59951   Some-college             10   \n",
       "34  22.0     State-gov  311512   Some-college             10   \n",
       "\n",
       "         marital-status        occupation    relationship  \\\n",
       "25   Married-civ-spouse      Tech-support         Husband   \n",
       "26        Never-married      Craft-repair       Own-child   \n",
       "27   Married-civ-spouse                 ?         Husband   \n",
       "28             Divorced   Exec-managerial   Not-in-family   \n",
       "29   Married-civ-spouse      Craft-repair         Husband   \n",
       "30        Never-married   Protective-serv   Not-in-family   \n",
       "31        Never-married             Sales       Own-child   \n",
       "32             Divorced   Exec-managerial       Own-child   \n",
       "33   Married-civ-spouse      Adm-clerical       Own-child   \n",
       "34   Married-civ-spouse     Other-service         Husband   \n",
       "\n",
       "                   race    sex  capital-gain  capital-loss  hours-per-week  \\\n",
       "25                White   Male             0             0              40   \n",
       "26                White   Male             0             0              40   \n",
       "27   Asian-Pac-Islander   Male             0             0              60   \n",
       "28                White   Male             0             0              80   \n",
       "29                White   Male             0             0              40   \n",
       "30                White   Male             0             0              52   \n",
       "31                Black   Male             0             0              44   \n",
       "32                White   Male             0          1408              40   \n",
       "33                White   Male             0             0              40   \n",
       "34                Black   Male             0             0              15   \n",
       "\n",
       "    native-country  \n",
       "25   United-States  \n",
       "26   United-States  \n",
       "27           South  \n",
       "28   United-States  \n",
       "29   United-States  \n",
       "30   United-States  \n",
       "31   United-States  \n",
       "32   United-States  \n",
       "33   United-States  \n",
       "34   United-States  "
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data5 = impute_outlier_with_avg(data, 'age', index, 'mean')\n",
    "data5[25:35]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "lesbian-flooring",
   "metadata": {},
   "source": [
    "# Шкалирование данных"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "personalized-heath",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((26048, 14), (6513, 14))"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(data, data.age, test_size=0.2, random_state=0)\n",
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cultural-intermediate",
   "metadata": {},
   "source": [
    "# Нормализация данных"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "threaded-nothing",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       age          workclass  fnlwgt      education  education-num  \\\n",
      "15282   36            Private  174308           11th              7   \n",
      "24870   35            Private  198202        HS-grad              9   \n",
      "18822   38            Private   52963      Bachelors             13   \n",
      "26404   50            Private  138270        HS-grad              9   \n",
      "7842    68   Self-emp-not-inc  116903      Assoc-voc             11   \n",
      "4890    51   Self-emp-not-inc  149220   Some-college             10   \n",
      "\n",
      "            marital-status         occupation    relationship    race  \\\n",
      "15282             Divorced   Transport-moving   Not-in-family   White   \n",
      "24870        Never-married    Exec-managerial   Not-in-family   White   \n",
      "18822        Never-married       Adm-clerical   Not-in-family   White   \n",
      "26404   Married-civ-spouse              Sales            Wife   Black   \n",
      "7842    Married-civ-spouse     Prof-specialty         Husband   White   \n",
      "4890    Married-civ-spouse    Farming-fishing         Husband   White   \n",
      "\n",
      "           sex  capital-gain  capital-loss  hours-per-week  native-country  \\\n",
      "15282     Male             0             0              40   United-States   \n",
      "24870   Female             0             0              54   United-States   \n",
      "18822   Female             0             0              50   United-States   \n",
      "26404   Female             0             0              40   United-States   \n",
      "7842      Male             0          2149              40   United-States   \n",
      "4890      Male             0             0              75   United-States   \n",
      "\n",
      "       age_zscore  \n",
      "15282   -0.189281  \n",
      "24870   -0.262660  \n",
      "18822   -0.042523  \n",
      "26404    0.838021  \n",
      "7842     2.158837  \n",
      "4890     0.911400  \n"
     ]
    }
   ],
   "source": [
    "ss = StandardScaler().fit(X_train[['age']])\n",
    "X_train_copy = X_train.copy()\n",
    "X_train_copy['age_zscore'] = ss.transform(X_train_copy[['age']])\n",
    "print(X_train_copy.head(6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "quantitative-yield",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.481206639733317e-16\n",
      "1.000019195884404\n"
     ]
    }
   ],
   "source": [
    "print(X_train_copy['age_zscore'].mean())\n",
    "print(X_train_copy['age_zscore'].std())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "earned-seven",
   "metadata": {},
   "source": [
    "Мин-макс шкалирование"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "double-motorcycle",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       age          workclass  fnlwgt      education  education-num  \\\n",
      "15282   36            Private  174308           11th              7   \n",
      "24870   35            Private  198202        HS-grad              9   \n",
      "18822   38            Private   52963      Bachelors             13   \n",
      "26404   50            Private  138270        HS-grad              9   \n",
      "7842    68   Self-emp-not-inc  116903      Assoc-voc             11   \n",
      "4890    51   Self-emp-not-inc  149220   Some-college             10   \n",
      "\n",
      "            marital-status         occupation    relationship    race  \\\n",
      "15282             Divorced   Transport-moving   Not-in-family   White   \n",
      "24870        Never-married    Exec-managerial   Not-in-family   White   \n",
      "18822        Never-married       Adm-clerical   Not-in-family   White   \n",
      "26404   Married-civ-spouse              Sales            Wife   Black   \n",
      "7842    Married-civ-spouse     Prof-specialty         Husband   White   \n",
      "4890    Married-civ-spouse    Farming-fishing         Husband   White   \n",
      "\n",
      "           sex  capital-gain  capital-loss  hours-per-week  native-country  \\\n",
      "15282     Male             0             0              40   United-States   \n",
      "24870   Female             0             0              54   United-States   \n",
      "18822   Female             0             0              50   United-States   \n",
      "26404   Female             0             0              40   United-States   \n",
      "7842      Male             0          2149              40   United-States   \n",
      "4890      Male             0             0              75   United-States   \n",
      "\n",
      "       age_minmax  \n",
      "15282    0.260274  \n",
      "24870    0.246575  \n",
      "18822    0.287671  \n",
      "26404    0.452055  \n",
      "7842     0.698630  \n",
      "4890     0.465753  \n"
     ]
    }
   ],
   "source": [
    "mms = MinMaxScaler().fit(X_train[['age']])\n",
    "X_train_copy = X_train.copy()\n",
    "X_train_copy['age_minmax'] = mms.transform(X_train_copy[['age']])\n",
    "print(X_train_copy.head(6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "statistical-throat",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0\n",
      "1.0\n"
     ]
    }
   ],
   "source": [
    "print(X_train_copy['age_minmax'].min())\n",
    "print(X_train_copy['age_minmax'].max())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sporting-marble",
   "metadata": {},
   "source": [
    "Робустное шкалирование"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "sticky-string",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       age          workclass  fnlwgt      education  education-num  \\\n",
      "15282   36            Private  174308           11th              7   \n",
      "24870   35            Private  198202        HS-grad              9   \n",
      "18822   38            Private   52963      Bachelors             13   \n",
      "26404   50            Private  138270        HS-grad              9   \n",
      "7842    68   Self-emp-not-inc  116903      Assoc-voc             11   \n",
      "4890    51   Self-emp-not-inc  149220   Some-college             10   \n",
      "\n",
      "            marital-status         occupation    relationship    race  \\\n",
      "15282             Divorced   Transport-moving   Not-in-family   White   \n",
      "24870        Never-married    Exec-managerial   Not-in-family   White   \n",
      "18822        Never-married       Adm-clerical   Not-in-family   White   \n",
      "26404   Married-civ-spouse              Sales            Wife   Black   \n",
      "7842    Married-civ-spouse     Prof-specialty         Husband   White   \n",
      "4890    Married-civ-spouse    Farming-fishing         Husband   White   \n",
      "\n",
      "           sex  capital-gain  capital-loss  hours-per-week  native-country  \\\n",
      "15282     Male             0             0              40   United-States   \n",
      "24870   Female             0             0              54   United-States   \n",
      "18822   Female             0             0              50   United-States   \n",
      "26404   Female             0             0              40   United-States   \n",
      "7842      Male             0          2149              40   United-States   \n",
      "4890      Male             0             0              75   United-States   \n",
      "\n",
      "       age_robust  \n",
      "15282   -0.052632  \n",
      "24870   -0.105263  \n",
      "18822    0.052632  \n",
      "26404    0.684211  \n",
      "7842     1.631579  \n",
      "4890     0.736842  \n"
     ]
    }
   ],
   "source": [
    "rs = RobustScaler().fit(X_train[['age']])\n",
    "X_train_copy = X_train.copy()\n",
    "X_train_copy['age_robust'] = rs.transform(X_train_copy[['age']])\n",
    "print(X_train_copy.head(6))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "norwegian-riding",
   "metadata": {},
   "source": [
    "# Энкодинг переменных"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "alert-queen",
   "metadata": {},
   "source": [
    "One-Hot encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "central-talent",
   "metadata": {},
   "outputs": [],
   "source": [
    "data1 = pd.get_dummies(data, drop_first=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "searching-sullivan",
   "metadata": {},
   "source": [
    "Энкодинг значением"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "detected-gasoline",
   "metadata": {},
   "outputs": [],
   "source": [
    "ord_enc = ce.OrdinalEncoder(cols=['sex']).fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "romantic-brother",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   age          workclass  fnlwgt   education  education-num  \\\n",
      "0   39          State-gov   77516   Bachelors             13   \n",
      "1   50   Self-emp-not-inc   83311   Bachelors             13   \n",
      "2   38            Private  215646     HS-grad              9   \n",
      "3   53            Private  234721        11th              7   \n",
      "4   28            Private  338409   Bachelors             13   \n",
      "\n",
      "        marital-status          occupation    relationship    race  sex  \\\n",
      "0        Never-married        Adm-clerical   Not-in-family   White    1   \n",
      "1   Married-civ-spouse     Exec-managerial         Husband   White    1   \n",
      "2             Divorced   Handlers-cleaners   Not-in-family   White    1   \n",
      "3   Married-civ-spouse   Handlers-cleaners         Husband   Black    1   \n",
      "4   Married-civ-spouse      Prof-specialty            Wife   Black    2   \n",
      "\n",
      "   capital-gain  capital-loss  hours-per-week  native-country  \n",
      "0          2174             0              40   United-States  \n",
      "1             0             0              13   United-States  \n",
      "2             0             0              40   United-States  \n",
      "3             0             0              40   United-States  \n",
      "4             0             0              40            Cuba  \n"
     ]
    }
   ],
   "source": [
    "data4 = ord_enc.transform(data)\n",
    "print(data4.head(5))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "major-carpet",
   "metadata": {},
   "source": [
    "Таргет энкодинг"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "spread-belle",
   "metadata": {},
   "outputs": [],
   "source": [
    "target_enc = ce.TargetEncoder(cols=['sex']).fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "duplicate-demonstration",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>workclass</th>\n",
       "      <th>fnlwgt</th>\n",
       "      <th>education</th>\n",
       "      <th>education-num</th>\n",
       "      <th>marital-status</th>\n",
       "      <th>occupation</th>\n",
       "      <th>relationship</th>\n",
       "      <th>race</th>\n",
       "      <th>sex</th>\n",
       "      <th>capital-gain</th>\n",
       "      <th>capital-loss</th>\n",
       "      <th>hours-per-week</th>\n",
       "      <th>native-country</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>39</td>\n",
       "      <td>State-gov</td>\n",
       "      <td>77516</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Adm-clerical</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>39.431971</td>\n",
       "      <td>2174</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>50</td>\n",
       "      <td>Self-emp-not-inc</td>\n",
       "      <td>83311</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Exec-managerial</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>39.431971</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>38</td>\n",
       "      <td>Private</td>\n",
       "      <td>215646</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>Handlers-cleaners</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>39.431971</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>53</td>\n",
       "      <td>Private</td>\n",
       "      <td>234721</td>\n",
       "      <td>11th</td>\n",
       "      <td>7</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Handlers-cleaners</td>\n",
       "      <td>Husband</td>\n",
       "      <td>Black</td>\n",
       "      <td>39.431971</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>Private</td>\n",
       "      <td>338409</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Prof-specialty</td>\n",
       "      <td>Wife</td>\n",
       "      <td>Black</td>\n",
       "      <td>36.852097</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>Cuba</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age          workclass  fnlwgt   education  education-num  \\\n",
       "0   39          State-gov   77516   Bachelors             13   \n",
       "1   50   Self-emp-not-inc   83311   Bachelors             13   \n",
       "2   38            Private  215646     HS-grad              9   \n",
       "3   53            Private  234721        11th              7   \n",
       "4   28            Private  338409   Bachelors             13   \n",
       "\n",
       "        marital-status          occupation    relationship    race        sex  \\\n",
       "0        Never-married        Adm-clerical   Not-in-family   White  39.431971   \n",
       "1   Married-civ-spouse     Exec-managerial         Husband   White  39.431971   \n",
       "2             Divorced   Handlers-cleaners   Not-in-family   White  39.431971   \n",
       "3   Married-civ-spouse   Handlers-cleaners         Husband   Black  39.431971   \n",
       "4   Married-civ-spouse      Prof-specialty            Wife   Black  36.852097   \n",
       "\n",
       "   capital-gain  capital-loss  hours-per-week  native-country  \n",
       "0          2174             0              40   United-States  \n",
       "1             0             0              13   United-States  \n",
       "2             0             0              40   United-States  \n",
       "3             0             0              40   United-States  \n",
       "4             0             0              40            Cuba  "
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data2 = target_enc.transform(data)\n",
    "data2.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "broadband-victoria",
   "metadata": {},
   "source": [
    "WOE энкодинг"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "engaged-palestinian",
   "metadata": {},
   "outputs": [],
   "source": [
    "# woe_enc = ce.WOEEncoder(cols=['sex']).fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "normal-austria",
   "metadata": {},
   "outputs": [],
   "source": [
    "# data3 = woe_enc.transform(data)\n",
    "# data3.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "behind-beginning",
   "metadata": {},
   "source": [
    "# Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "neither-scholarship",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_breast_cancer\n",
    "data = load_breast_cancer()\n",
    "data = pd.DataFrame(np.c_[data['data'], data['target']], columns=np.append(data['feature_names'], ['target']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "exclusive-privilege",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean radius</th>\n",
       "      <th>mean texture</th>\n",
       "      <th>mean perimeter</th>\n",
       "      <th>mean area</th>\n",
       "      <th>mean smoothness</th>\n",
       "      <th>mean compactness</th>\n",
       "      <th>mean concavity</th>\n",
       "      <th>mean concave points</th>\n",
       "      <th>mean symmetry</th>\n",
       "      <th>mean fractal dimension</th>\n",
       "      <th>...</th>\n",
       "      <th>worst texture</th>\n",
       "      <th>worst perimeter</th>\n",
       "      <th>worst area</th>\n",
       "      <th>worst smoothness</th>\n",
       "      <th>worst compactness</th>\n",
       "      <th>worst concavity</th>\n",
       "      <th>worst concave points</th>\n",
       "      <th>worst symmetry</th>\n",
       "      <th>worst fractal dimension</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17.99</td>\n",
       "      <td>10.38</td>\n",
       "      <td>122.80</td>\n",
       "      <td>1001.0</td>\n",
       "      <td>0.11840</td>\n",
       "      <td>0.27760</td>\n",
       "      <td>0.3001</td>\n",
       "      <td>0.14710</td>\n",
       "      <td>0.2419</td>\n",
       "      <td>0.07871</td>\n",
       "      <td>...</td>\n",
       "      <td>17.33</td>\n",
       "      <td>184.60</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>0.1622</td>\n",
       "      <td>0.6656</td>\n",
       "      <td>0.7119</td>\n",
       "      <td>0.2654</td>\n",
       "      <td>0.4601</td>\n",
       "      <td>0.11890</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20.57</td>\n",
       "      <td>17.77</td>\n",
       "      <td>132.90</td>\n",
       "      <td>1326.0</td>\n",
       "      <td>0.08474</td>\n",
       "      <td>0.07864</td>\n",
       "      <td>0.0869</td>\n",
       "      <td>0.07017</td>\n",
       "      <td>0.1812</td>\n",
       "      <td>0.05667</td>\n",
       "      <td>...</td>\n",
       "      <td>23.41</td>\n",
       "      <td>158.80</td>\n",
       "      <td>1956.0</td>\n",
       "      <td>0.1238</td>\n",
       "      <td>0.1866</td>\n",
       "      <td>0.2416</td>\n",
       "      <td>0.1860</td>\n",
       "      <td>0.2750</td>\n",
       "      <td>0.08902</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19.69</td>\n",
       "      <td>21.25</td>\n",
       "      <td>130.00</td>\n",
       "      <td>1203.0</td>\n",
       "      <td>0.10960</td>\n",
       "      <td>0.15990</td>\n",
       "      <td>0.1974</td>\n",
       "      <td>0.12790</td>\n",
       "      <td>0.2069</td>\n",
       "      <td>0.05999</td>\n",
       "      <td>...</td>\n",
       "      <td>25.53</td>\n",
       "      <td>152.50</td>\n",
       "      <td>1709.0</td>\n",
       "      <td>0.1444</td>\n",
       "      <td>0.4245</td>\n",
       "      <td>0.4504</td>\n",
       "      <td>0.2430</td>\n",
       "      <td>0.3613</td>\n",
       "      <td>0.08758</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.42</td>\n",
       "      <td>20.38</td>\n",
       "      <td>77.58</td>\n",
       "      <td>386.1</td>\n",
       "      <td>0.14250</td>\n",
       "      <td>0.28390</td>\n",
       "      <td>0.2414</td>\n",
       "      <td>0.10520</td>\n",
       "      <td>0.2597</td>\n",
       "      <td>0.09744</td>\n",
       "      <td>...</td>\n",
       "      <td>26.50</td>\n",
       "      <td>98.87</td>\n",
       "      <td>567.7</td>\n",
       "      <td>0.2098</td>\n",
       "      <td>0.8663</td>\n",
       "      <td>0.6869</td>\n",
       "      <td>0.2575</td>\n",
       "      <td>0.6638</td>\n",
       "      <td>0.17300</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20.29</td>\n",
       "      <td>14.34</td>\n",
       "      <td>135.10</td>\n",
       "      <td>1297.0</td>\n",
       "      <td>0.10030</td>\n",
       "      <td>0.13280</td>\n",
       "      <td>0.1980</td>\n",
       "      <td>0.10430</td>\n",
       "      <td>0.1809</td>\n",
       "      <td>0.05883</td>\n",
       "      <td>...</td>\n",
       "      <td>16.67</td>\n",
       "      <td>152.20</td>\n",
       "      <td>1575.0</td>\n",
       "      <td>0.1374</td>\n",
       "      <td>0.2050</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.1625</td>\n",
       "      <td>0.2364</td>\n",
       "      <td>0.07678</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean radius  mean texture  mean perimeter  mean area  mean smoothness  \\\n",
       "0        17.99         10.38          122.80     1001.0          0.11840   \n",
       "1        20.57         17.77          132.90     1326.0          0.08474   \n",
       "2        19.69         21.25          130.00     1203.0          0.10960   \n",
       "3        11.42         20.38           77.58      386.1          0.14250   \n",
       "4        20.29         14.34          135.10     1297.0          0.10030   \n",
       "\n",
       "   mean compactness  mean concavity  mean concave points  mean symmetry  \\\n",
       "0           0.27760          0.3001              0.14710         0.2419   \n",
       "1           0.07864          0.0869              0.07017         0.1812   \n",
       "2           0.15990          0.1974              0.12790         0.2069   \n",
       "3           0.28390          0.2414              0.10520         0.2597   \n",
       "4           0.13280          0.1980              0.10430         0.1809   \n",
       "\n",
       "   mean fractal dimension  ...  worst texture  worst perimeter  worst area  \\\n",
       "0                 0.07871  ...          17.33           184.60      2019.0   \n",
       "1                 0.05667  ...          23.41           158.80      1956.0   \n",
       "2                 0.05999  ...          25.53           152.50      1709.0   \n",
       "3                 0.09744  ...          26.50            98.87       567.7   \n",
       "4                 0.05883  ...          16.67           152.20      1575.0   \n",
       "\n",
       "   worst smoothness  worst compactness  worst concavity  worst concave points  \\\n",
       "0            0.1622             0.6656           0.7119                0.2654   \n",
       "1            0.1238             0.1866           0.2416                0.1860   \n",
       "2            0.1444             0.4245           0.4504                0.2430   \n",
       "3            0.2098             0.8663           0.6869                0.2575   \n",
       "4            0.1374             0.2050           0.4000                0.1625   \n",
       "\n",
       "   worst symmetry  worst fractal dimension  target  \n",
       "0          0.4601                  0.11890     0.0  \n",
       "1          0.2750                  0.08902     0.0  \n",
       "2          0.3613                  0.08758     0.0  \n",
       "3          0.6638                  0.17300     0.0  \n",
       "4          0.2364                  0.07678     0.0  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "rural-class",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((455, 30), (114, 30))"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(data.drop(labels=['target'], axis=1),\n",
    "                                                   data.target, test_size=0.2,\n",
    "                                                   random_state=0)\n",
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "boolean-baltimore",
   "metadata": {},
   "outputs": [],
   "source": [
    "def constant_feature_detect(data, threshold=0.98):\n",
    "    data_copy = data.copy()\n",
    "    quasi_constant_feature = []\n",
    "    for feature in data_copy.columns:\n",
    "        predomiant = (data_copy[feature].value_counts() / np.float(\n",
    "                     len(data_copy))).sort_values(ascending=False).values[0]\n",
    "        if predomiant >= threshold:\n",
    "            quasi_constant_feature.append(feature)\n",
    "    print(len(quasi_constant_feature), 'константные переменные')\n",
    "    return quasi_constant_feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "danish-format",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 константные переменные\n"
     ]
    }
   ],
   "source": [
    "quasi_constant_feature = constant_feature_detect(data=X_train, threshold=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "presidential-sound",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0    0.923077\n",
       "0.0    0.068132\n",
       "2.0    0.008791\n",
       "Name: dummy, dtype: float64"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train['dummy'] = np.floor(X_train['worst smoothness']*10)\n",
    "X_train.dummy.value_counts() / np.float(len(X_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "coastal-roman",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 константные переменные\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['dummy']"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quasi_constant_feature = constant_feature_detect(data=X_train, threshold=0.9)\n",
    "quasi_constant_feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "recovered-contribution",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(455, 30)\n"
     ]
    }
   ],
   "source": [
    "X_train.drop(labels=quasi_constant_feature, axis=1, inplace=True)\n",
    "print(X_train.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "advisory-knife",
   "metadata": {},
   "source": [
    "Корреляционная фильтрация"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "continuing-ethernet",
   "metadata": {},
   "outputs": [],
   "source": [
    "def corr_feature_detect(data, threshold=0.8):\n",
    "    \n",
    "    corrmat = data.corr()\n",
    "    corrmat = corrmat.abs().unstack()\n",
    "    corrmat = corrmat.sort_values(ascending=False)\n",
    "    corrmat = corrmat[corrmat >= threshold]\n",
    "    corrmat = corrmat[corrmat < 1]\n",
    "    corrmat = pd.DataFrame(corrmat).reset_index()\n",
    "    corrmat.columns = ['feature1', 'feature2', 'corr']\n",
    "    \n",
    "    grouped_feature_ls = []\n",
    "    correlated_groups = []\n",
    "    \n",
    "    for feature in corrmat.feature1.unique():\n",
    "        if feature not in grouped_feature_ls:\n",
    "            \n",
    "            correlation_block = corrmat[corrmat.feature1 == feature]\n",
    "            grouped_feature_ls = grouped_feature_ls + list(\n",
    "                correlation_block.feature2.unique()) + [feature]\n",
    "            \n",
    "            correlated_groups.append(correlation_block)\n",
    "    return correlated_groups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "oriented-litigation",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          feature1         feature2      corr\n",
      "0   mean perimeter      mean radius  0.998185\n",
      "7   mean perimeter        mean area  0.986692\n",
      "14  mean perimeter  worst perimeter  0.970507\n",
      "19  mean perimeter     worst radius  0.969520\n",
      "32  mean perimeter       worst area  0.941920 \n",
      "\n",
      "           feature1      feature2      corr\n",
      "12  perimeter error  radius error  0.978323\n",
      "30  perimeter error    area error  0.944995 \n",
      "\n",
      "               feature1              feature2      corr\n",
      "36  mean concave points        mean concavity  0.914627\n",
      "41  mean concave points  worst concave points  0.906312 \n",
      "\n",
      "         feature1      feature2      corr\n",
      "38  worst texture  mean texture  0.908182 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "corr = corr_feature_detect(data=X_train, threshold=0.9)\n",
    "for i in corr:\n",
    "    print(i, '\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "attempted-assurance",
   "metadata": {},
   "source": [
    "Взаимная информация"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "trying-methodology",
   "metadata": {},
   "outputs": [],
   "source": [
    "def mutual_info(X, y, select_k=10):\n",
    "    \n",
    "    if select_k >= 1:\n",
    "        sel_ = SelectKBest(mutual_info_classif, k=select_k).fit(X, y)\n",
    "        col = X.columns[sel_.get_support()]\n",
    "    \n",
    "    elif 0 < select_k < 1:\n",
    "        sel_ = SelectPercentile(mutual_info_classif, percentile=select_k*100).fit(X, y)\n",
    "        col = X.columns[sel_.get_support()]\n",
    "    \n",
    "    else:\n",
    "        raise ValueError('select_k должно быть положительным значением')\n",
    "    \n",
    "    return col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "otherwise-beauty",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['mean concave points', 'worst perimeter', 'worst area'], dtype='object')\n"
     ]
    }
   ],
   "source": [
    "mi = mutual_info(X=X_train, y=y_train, select_k=3)\n",
    "print(mi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "silent-mississippi",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['mean perimeter', 'mean concave points', 'worst radius',\n",
      "       'worst perimeter', 'worst area', 'worst concave points'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "mi = mutual_info(X=X_train, y=y_train, select_k=0.2)\n",
    "print(mi)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "personalized-examination",
   "metadata": {},
   "source": [
    "Хи-квадрат тест"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "honest-oasis",
   "metadata": {},
   "outputs": [],
   "source": [
    "def chi_square_test(X, y, select_k=10):\n",
    "    \n",
    "    if select_k >= 1:\n",
    "        sel_ = SelectKBest(chi2, k=select_k).fit(X, y)\n",
    "        col = X.columns[sel_.get_support()]\n",
    "        \n",
    "    elif 0 < select_k < 1:\n",
    "        sel_ = SelectPercentile(chi2, percentile=select_k*100).fit(X, y)\n",
    "        col = X.columns[sel_.get_support()]\n",
    "    \n",
    "    else:\n",
    "        raise ValueError('select_k должно быть положительным значением')\n",
    "    \n",
    "    return col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "increased-highway",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['mean area', 'area error', 'worst area'], dtype='object')\n"
     ]
    }
   ],
   "source": [
    "chi = chi_square_test(X_train, y_train, 3)\n",
    "print(chi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "southwest-notebook",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['mean perimeter', 'mean area', 'area error', 'worst radius',\n",
      "       'worst perimeter', 'worst area'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "chi = chi_square_test(X_train, y_train, 0.2)\n",
    "print(chi)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cheap-singing",
   "metadata": {},
   "source": [
    " Одномерный ROC-AUC(если задача классификации) или MSE (если задача регрессии)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "authentic-thumbnail",
   "metadata": {},
   "outputs": [],
   "source": [
    "def univariate_roc_auc(X_train, y_train, X_test, y_test, threshold):\n",
    "    \n",
    "    roc_values = []\n",
    "    for feature in X_train.columns:\n",
    "        clf = DecisionTreeClassifier()\n",
    "        clf.fit(X_train[feature].to_frame(), y_train)\n",
    "        y_scored = clf.predict_proba(X_test[feature].to_frame())\n",
    "        roc_values.append(roc_auc_score(y_test, y_scored[:, 1]))\n",
    "    roc_values = pd.Series(roc_values)\n",
    "    roc_values.index = X_train.columns\n",
    "    print(roc_values.sort_values(ascending=False))\n",
    "    print(len(roc_values[roc_values > threshold]), len(X_train.columns))\n",
    "    keep_col = roc_values[roc_values > threshold]\n",
    "    return keep_col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "precious-attribute",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "worst perimeter            0.917275\n",
      "worst area                 0.895840\n",
      "worst radius               0.893458\n",
      "worst concave points       0.863131\n",
      "mean concavity             0.856939\n",
      "mean radius                0.849000\n",
      "mean area                  0.839314\n",
      "worst concavity            0.831375\n",
      "mean perimeter             0.829628\n",
      "mean concave points        0.826453\n",
      "area error                 0.812321\n",
      "worst compactness          0.742299\n",
      "radius error               0.740235\n",
      "mean compactness           0.734360\n",
      "perimeter error            0.680534\n",
      "worst texture              0.647666\n",
      "worst fractal dimension    0.640997\n",
      "concavity error            0.640203\n",
      "worst symmetry             0.620991\n",
      "concave points error       0.618133\n",
      "compactness error          0.607336\n",
      "mean symmetry              0.591775\n",
      "mean texture               0.573357\n",
      "texture error              0.568593\n",
      "worst smoothness           0.565100\n",
      "mean smoothness            0.557637\n",
      "fractal dimension error    0.542077\n",
      "smoothness error           0.522706\n",
      "symmetry error             0.493649\n",
      "mean fractal dimension     0.475548\n",
      "dtype: float64\n",
      "11 30\n",
      "mean radius             0.849000\n",
      "mean perimeter          0.829628\n",
      "mean area               0.839314\n",
      "mean concavity          0.856939\n",
      "mean concave points     0.826453\n",
      "area error              0.812321\n",
      "worst radius            0.893458\n",
      "worst perimeter         0.917275\n",
      "worst area              0.895840\n",
      "worst concavity         0.831375\n",
      "worst concave points    0.863131\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "uni_roc_auc = univariate_roc_auc(X_train, y_train, X_test, y_test, threshold=0.8)\n",
    "print(uni_roc_auc)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "considered-talent",
   "metadata": {},
   "source": [
    "# Oversampling/undersampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "legal-evaluation",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "455"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "sporting-douglas",
   "metadata": {},
   "outputs": [],
   "source": [
    "sm = SMOTE(random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "grateful-amazon",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_res, y_res = sm.fit_resample(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "amateur-douglas",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "580"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(y_res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "subsequent-toolbox",
   "metadata": {},
   "outputs": [],
   "source": [
    "cn = CondensedNearestNeighbour(random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "facial-situation",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_res, y_res = cn.fit_resample(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "automotive-cutting",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "205"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(y_res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "modified-romance",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}