Image_Classification_DAISY_5_roc_auc.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Image Classification "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import datetime\n",
    "import pickle\n",
    "import numpy as np\n",
    "from sklearn import svm, metrics\n",
    "from sklearn.model_selection import StratifiedKFold,GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from pandas import DataFrame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Dataset from the pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start time:  2019-01-18 11:41:22.160393\n"
     ]
    }
   ],
   "source": [
    "# Start point\n",
    "start_time = time.time()\n",
    "print(\"Start time: \", datetime.datetime.now())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle_in = open(\"Signature_Dataset_DAISY_110_5_int.pickle\",\"rb\")\n",
    "image_dataset = pickle.load(pickle_in)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split data in Training Set and Testing Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----Dataset Loaded From Pickle----\n"
     ]
    }
   ],
   "source": [
    "X_train = image_dataset.X_train\n",
    "X_test = image_dataset.X_test\n",
    "y_train = image_dataset.y_train\n",
    "y_test = image_dataset.y_test\n",
    "\n",
    "print(\"----Dataset Loaded From Pickle----\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train data with parameter optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Dictionary of models and hyperparameters\n",
    "\n",
    "models_and_hyperparameters={'LR':(LogisticRegression(),{'C': [0.01, 0.05, 0.1, 0.5, 1, 2],\n",
    "                                                        'penalty': ['l1', 'l2'], \n",
    "                                                        'class_weight': ['balanced'] }),\n",
    "                           'SVM':(svm.SVC(),{'C': [1, 10, 100, 1000],\n",
    "                                             'gamma': [0.001, 0.0001],\n",
    "                                             'kernel': ['rbf'],\n",
    "                                             'class_weight': ['balanced']}),\n",
    "                           'RFC':(RandomForestClassifier(), {'n_estimators': [50,100,200],\n",
    "                                                             # 'max_features': ['auto', 'sqrt'],\n",
    "                                                             'max_depth': [50,60,70],\n",
    "                                                             'min_samples_split': [2,5,10],\n",
    "                                                             #'min_samples_leaf': [1,2],\n",
    "                                                             'bootstrap': [True, False],\n",
    "                                                             'class_weight': ['balanced']})\n",
    "                           }\n",
    "\n",
    "#from the run with 4 sign we estimated the best parameters for the RFC\n",
    "\n",
    "best_hyperparameters= {'bootstrap': [False], \n",
    "                       'class_weight': ['balanced'], \n",
    "                       'max_depth': [70],\n",
    "                       'min_samples_split': [5], \n",
    "                       'n_estimators': [200]}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #LOGISTIC REGRESSION\n",
    "\n",
    "# print(\"----Training----\")\n",
    "\n",
    "# k_fold = StratifiedKFold(n_splits=10, shuffle=True, random_state=31)\n",
    "\n",
    "# model = models_and_hyperparameters['LR'][0]\n",
    "# hyperparameters = models_and_hyperparameters['LR'][1]\n",
    "\n",
    "# gs = GridSearchCV(model, \n",
    "#                    param_grid = hyperparameters,\n",
    "#                    cv=k_fold, \n",
    "#                    scoring='roc_auc',\n",
    "#                    verbose=3,\n",
    "#                    return_train_score=True)\n",
    "\n",
    "# gs.fit(X_train, y_train)\n",
    "\n",
    "# print(\"----Training Ended----\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#DataFrame(gs.cv_results_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Creation of the pickle which is goig to store the model.\n",
    "\n",
    "# pickle_out = open(\"LR_roc_auc_DAISY_3.pickle\",\"wb\") #Modify with the name of the dataset you're saving\n",
    "# pickle.dump(gs.cv_results_, pickle_out)\n",
    "# pickle_out.close()\n",
    "\n",
    "# print(\"----Pickle created----\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y_pred = gs.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"Best hyperparameters: {}\".format(gs.best_params_))\n",
    "# print(\"Best roc_auc score: {:3f}\".format(gs.best_score_))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(\"Classification report for - \\n{}:\\n{}\\n\".format(gs, metrics.classification_report(y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\\\\n",
    "\\\\\n",
    "\\\\\n",
    "\\\\\n",
    "\\\\\n",
    "# SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# # SVM\n",
    "\n",
    "# print(\"----Training----\")\n",
    "\n",
    "# k_fold = StratifiedKFold(n_splits=4, shuffle=True, random_state=31)\n",
    "\n",
    "# model = models_and_hyperparameters['SVM'][0]\n",
    "# hyperparameters = models_and_hyperparameters['SVM'][1]\n",
    "\n",
    "# gs = GridSearchCV(model, \n",
    "#                    param_grid = hyperparameters,\n",
    "#                    cv=k_fold, \n",
    "#                    scoring='roc_auc',\n",
    "#                    verbose=3,\n",
    "#                    return_train_score=True)\n",
    "\n",
    "# gs.fit(X_train, y_train)\n",
    "\n",
    "# print(\"----Training Ended----\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DataFrame(gs.cv_results_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Creation of the pickle which is goig to store the model.\n",
    "\n",
    "# pickle_out = open(\"SVM_roc_auc_DAISY_3.pickle\",\"wb\") #Modify with the name of the dataset you're saving\n",
    "# pickle.dump(gs.cv_results_, pickle_out)\n",
    "# pickle_out.close()\n",
    "\n",
    "# print(\"----Pickle created----\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y_pred = gs.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"Best hyperparameters: {}\".format(gs.best_params_))\n",
    "# print(\"Best roc_auc score: {:3f}\".format(gs.best_score_))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"Classification report for - \\n{}:\\n{}\\n\".format(gs, metrics.classification_report(y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "//\n",
    "//\n",
    "//\n",
    "//\n",
    "//\n",
    "\n",
    "# Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----Training----\n",
      "Fitting 4 folds for each of 1 candidates, totalling 4 fits\n",
      "[CV] bootstrap=False, class_weight=balanced, max_depth=70, min_samples_split=5, n_estimators=200 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  bootstrap=False, class_weight=balanced, max_depth=70, min_samples_split=5, n_estimators=200, score=0.9324517906336088, total=  34.2s\n",
      "[CV] bootstrap=False, class_weight=balanced, max_depth=70, min_samples_split=5, n_estimators=200 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:   34.6s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  bootstrap=False, class_weight=balanced, max_depth=70, min_samples_split=5, n_estimators=200, score=0.9082534435261708, total=  31.2s\n",
      "[CV] bootstrap=False, class_weight=balanced, max_depth=70, min_samples_split=5, n_estimators=200 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:  1.1min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  bootstrap=False, class_weight=balanced, max_depth=70, min_samples_split=5, n_estimators=200, score=0.90633608815427, total=  31.7s\n",
      "[CV] bootstrap=False, class_weight=balanced, max_depth=70, min_samples_split=5, n_estimators=200 \n",
      "[CV]  bootstrap=False, class_weight=balanced, max_depth=70, min_samples_split=5, n_estimators=200, score=0.9154380165289256, total=  31.6s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:  2.2min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----Training Ended----\n"
     ]
    }
   ],
   "source": [
    "#RANDOM FOREST\n",
    "\n",
    "print(\"----Training----\")\n",
    "\n",
    "k_fold = StratifiedKFold(n_splits=4, shuffle=True, random_state=31)\n",
    "\n",
    "model = models_and_hyperparameters['RFC'][0]\n",
    "hyperparameters = best_hyperparameters\n",
    "\n",
    "gs = GridSearchCV(model, \n",
    "                   param_grid = hyperparameters,\n",
    "                   cv=k_fold, \n",
    "                   scoring='roc_auc',\n",
    "                   verbose=3,\n",
    "                   return_train_score=True)\n",
    "\n",
    "gs.fit(X_train, y_train)\n",
    "\n",
    "print(\"----Training Ended----\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>param_bootstrap</th>\n",
       "      <th>param_class_weight</th>\n",
       "      <th>param_max_depth</th>\n",
       "      <th>param_min_samples_split</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>params</th>\n",
       "      <th>...</th>\n",
       "      <th>split3_test_score</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "      <th>split0_train_score</th>\n",
       "      <th>split1_train_score</th>\n",
       "      <th>split2_train_score</th>\n",
       "      <th>split3_train_score</th>\n",
       "      <th>mean_train_score</th>\n",
       "      <th>std_train_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>32.100253</td>\n",
       "      <td>1.171545</td>\n",
       "      <td>0.097093</td>\n",
       "      <td>0.022713</td>\n",
       "      <td>False</td>\n",
       "      <td>balanced</td>\n",
       "      <td>70</td>\n",
       "      <td>5</td>\n",
       "      <td>200</td>\n",
       "      <td>{'bootstrap': False, 'class_weight': 'balanced...</td>\n",
       "      <td>...</td>\n",
       "      <td>0.915438</td>\n",
       "      <td>0.91562</td>\n",
       "      <td>0.010293</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_fit_time  std_fit_time  mean_score_time  std_score_time  \\\n",
       "0      32.100253      1.171545         0.097093        0.022713   \n",
       "\n",
       "  param_bootstrap param_class_weight param_max_depth param_min_samples_split  \\\n",
       "0           False           balanced              70                       5   \n",
       "\n",
       "  param_n_estimators                                             params  \\\n",
       "0                200  {'bootstrap': False, 'class_weight': 'balanced...   \n",
       "\n",
       "        ...         split3_test_score  mean_test_score  std_test_score  \\\n",
       "0       ...                  0.915438          0.91562        0.010293   \n",
       "\n",
       "   rank_test_score  split0_train_score  split1_train_score  \\\n",
       "0                1                 1.0                 1.0   \n",
       "\n",
       "   split2_train_score  split3_train_score  mean_train_score  std_train_score  \n",
       "0                 1.0                 1.0               1.0              0.0  \n",
       "\n",
       "[1 rows x 23 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DataFrame(gs.cv_results_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----Pickle created----\n"
     ]
    }
   ],
   "source": [
    "# Creation of the pickle which is goig to store the model.\n",
    "\n",
    "pickle_out = open(\"RF_roc_auc_DAISY_5.pickle\",\"wb\") #Modify with the name of the dataset you're saving\n",
    "pickle.dump(gs.cv_results_, pickle_out)\n",
    "pickle_out.close()\n",
    "\n",
    "print(\"----Pickle created----\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = gs.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best hyperparameters: {'bootstrap': False, 'class_weight': 'balanced', 'max_depth': 70, 'min_samples_split': 5, 'n_estimators': 200}\n",
      "Best roc_auc score: 0.915620\n"
     ]
    }
   ],
   "source": [
    "print(\"Best hyperparameters: {}\".format(gs.best_params_))\n",
    "print(\"Best roc_auc score: {:3f}\".format(gs.best_score_))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report for - \n",
      "GridSearchCV(cv=StratifiedKFold(n_splits=4, random_state=31, shuffle=True),\n",
      "       error_score='raise-deprecating',\n",
      "       estimator=RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
      "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
      "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "            min_samples_leaf=1, min_samples_split=2,\n",
      "            min_weight_fraction_leaf=0.0, n_estimators='warn', n_jobs=None,\n",
      "            oob_score=False, random_state=None, verbose=0,\n",
      "            warm_start=False),\n",
      "       fit_params=None, iid='warn', n_jobs=None,\n",
      "       param_grid={'bootstrap': [False], 'class_weight': ['balanced'], 'max_depth': [70], 'min_samples_split': [5], 'n_estimators': [200]},\n",
      "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
      "       scoring='roc_auc', verbose=3):\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -1       0.68      0.82      0.75       480\n",
      "           1       0.55      0.35      0.43       288\n",
      "\n",
      "   micro avg       0.65      0.65      0.65       768\n",
      "   macro avg       0.61      0.59      0.59       768\n",
      "weighted avg       0.63      0.65      0.63       768\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"Classification report for - \\n{}:\\n{}\\n\".format(gs, metrics.classification_report(y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "End time:  2019-01-18 11:44:20.397029\n",
      "Uptime : 0:02:58.236584\n"
     ]
    }
   ],
   "source": [
    "# End point\n",
    "end_time = time.time()\n",
    "uptime = end_time - start_time\n",
    "human_uptime = datetime.timedelta(seconds=uptime)\n",
    "\n",
    "print(\"End time: \", datetime.datetime.now())\n",
    "print(\"Uptime :\" ,human_uptime)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}