Model_train_realdt.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy.stats import norm\n",
    "import random\n",
    "import statistics as st\n",
    "from matplotlib import pyplot as plt\n",
    "import seaborn as sns\n",
    "import scipy as sp\n",
    "from scipy import stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def zipflaw(dt):\n",
    "    zipf_dt = pd.DataFrame(columns=['Frequency'])\n",
    "    nov_freq = dt.drop(['Author'])\n",
    "    temp2=nov_freq[nov_freq!=0].sort_values(ascending=False).reset_index()\n",
    "    frequency = temp2[temp2.columns[1]]\n",
    "    probability = frequency/np.sum(frequency)\n",
    "    zipf_dt['Frequency'] = frequency\n",
    "    zipf_dt['Probability'] = probability\n",
    "    rank=np.array(range(1,len(temp2)+1))\n",
    "    zipf_dt['rank'] = rank\n",
    "    initial = (1,1)\n",
    "    s_best = sp.optimize.minimize(logML_mbt, initial)\n",
    "    alpha=s_best.x[0]\n",
    "    beta = s_best.x[1]\n",
    "    zipf_dt['alpha'] = alpha\n",
    "    zipf_dt['beta'] = beta\n",
    "    zipf_dt['zipflaw']=((rank+beta)**(-alpha))/np.sum(1/(rank+beta)**alpha)\n",
    "    zipf_dt['log_pro'] = np.log(probability.astype(np.float64))\n",
    "    zipf_dt['log_rank'] = np.log(rank)\n",
    "    zipf_dt['log_zipflaw'] =  -alpha*np.log(rank+beta) - np.log(np.sum(1/(rank+beta)**alpha))\n",
    "    return zipf_dt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i in range (1,600):\n",
    "    dev=novice_dt.iloc[i]\n",
    "    nov_freq = dev.drop(['Author'])\n",
    "    temp2=nov_freq[nov_freq!=0].sort_values(ascending=False).reset_index()\n",
    "    frequency = temp2[temp2.columns[1]]\n",
    "    rank=np.array(range(1,len(temp2)+1))\n",
    "    initial = (1,1)\n",
    "    s_best = sp.optimize.minimize(logML_mbt, initial)\n",
    "    zipf_novice=zipf_novice.append({'alpha':s_best.x[0],'beta':s_best.x[1]},ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "zipf_expert = pd.DataFrame(columns=['alpha','beta','sample_size','rank'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i in range (1,600):\n",
    "    dev=expert_dt.iloc[i]\n",
    "    exp_freq = dev.drop(['Author'])\n",
    "    temp2=exp_freq[exp_freq!=0].sort_values(ascending=False).reset_index()\n",
    "    frequency = temp2[temp2.columns[1]]\n",
    "    rank=np.array(range(1,len(temp2)+1))\n",
    "    initial = (1,1)\n",
    "    s_best = sp.optimize.minimize(logML_mbt, initial)\n",
    "    zipf_expert=zipf_expert.append({'alpha':s_best.x[0],'beta':s_best.x[1]},ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#generate synthetic data for confidence interval\n",
    "for i in range(1,198):\n",
    "    rank=np.array(range(1,i))\n",
    "    zipf = pd.DataFrame(columns=['zipflaw'])\n",
    "    in_alpha=1.00001\n",
    "    zipf['zipflaw']=((rank)**(-in_alpha))/np.sum(1/(rank)**in_alpha)\n",
    "    test=zipf*1156\n",
    "    zipf_dt = pd.DataFrame(columns=['Frequency'])\n",
    "    #dev = dt_gby.drop(['Author'], axis=1)\n",
    "    #temp = dev.transpose()\n",
    "    #exp_freq = temp[0]\n",
    "    #temp2=exp_freq[exp_freq!=0].sort_values(ascending=False).reset_index()\n",
    "    df=test.astype(int)\n",
    "    #frequency = temp2[temp2.columns[1]]\n",
    "    frequency=df['zipflaw']\n",
    "    probability = frequency/np.sum(frequency)\n",
    "    zipf_dt['Frequency'] = frequency\n",
    "    #rank=np.array(range(1,len(temp2)+1))\n",
    "    zipf_dt['rank'] = rank\n",
    "    s_best = sp.optimize.minimize(logML, [1])\n",
    "    alpha=s_best.x\n",
    "    zipf_dt['alpha'] = alpha[0]\n",
    "    zipf_dt['zipflaw']=(rank**(-alpha))/np.sum(1/rank**alpha)\n",
    "    zipf_dt['zipf_freq']=zipf_dt['zipflaw']*np.sum(zipf_dt['Frequency'])\n",
    "    initial = (1,0)\n",
    "    mbt_best = sp.optimize.minimize(logML_mbt, initial)\n",
    "    alpha_mbt=mbt_best.x[0]\n",
    "    beta = mbt_best.x[1]\n",
    "    zipf_dt['alpha_mbt'] = alpha_mbt\n",
    "    zipf_dt['beta_mbt'] = beta\n",
    "    zipf_dt['zipf_mbt']=((rank+beta)**(-alpha_mbt))/np.sum(1/(rank+beta)**alpha_mbt)\n",
    "    zipf_dt['zipf_mbt_freq']=zipf_dt['zipf_mbt']*np.sum(zipf_dt['Frequency'])\n",
    "    zipf_dt['Probability'] = probability\n",
    "    zipf_dt['log_pro'] = np.log(probability.astype(np.float64))\n",
    "    zipf_dt['log_rank'] = np.log(rank)\n",
    "    zipf_dt['log_zipflaw'] = -alpha*np.log(rank) - np.log(np.sum(1/rank**alpha))\n",
    "    zipf_dt['log_zipf_mbt'] =  -alpha_mbt*np.log(rank+beta) - np.log(np.sum(1/(rank+beta)**alpha_mbt))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3020,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def token_probability(data):\n",
    "    token_pro = pd.DataFrame(columns=['token','pro'])\n",
    "    for col in data:\n",
    "        pro=np.sum(data[col])/np.sum(data.sum())\n",
    "        token_pro = token_pro.append({'token':col, 'pro': pro},ignore_index=True)\n",
    "    return token_pro"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3021,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nov_dt = cv_token.drop(['Author'],axis=1)\n",
    "pro_nov = token_probability(nov_dt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3225,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dt_freq = pd.DataFrame (pr_zpf,columns=['zipflaw'])   \n",
    "df=pr_zpf.astype(int)\n",
    "exp_freq=df\n",
    "temp2=exp_freq[exp_freq!=0].sort_values(ascending=False).reset_index()\n",
    "frequency = temp2[temp2.columns[1]]\n",
    "rank=np.array(range(1,len(temp2)+1))\n",
    "s_best = sp.optimize.minimize(logML, [1])\n",
    "alpha=s_best.x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3435,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dt_dev_kmeans_gy = dt_dev_kmeans.groupby('Author').sum().reset_index()\n",
    "#dt_dev_kmeans_nAuthor = dt_dev_kmeans_gy.drop(['Author'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3446,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dt_dev_kmeans_label=pd.merge(dt_dev_kmeans_gy,  \n",
    "                      vloop,  \n",
    "                      on ='Author',  \n",
    "                      how ='inner')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3418,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "GH_Data.to_csv(r'C:\\GH-Dataset\\GH_Data_alpha.csv', index=None, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2621,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "d_ks=max(abs(cdf_real-cdf_predict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3451,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3582,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3583,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vectorizer= TfidfVectorizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3584,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "response=vectorizer.fit_transform(dt_dev_kmeans_nAuthor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3573,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tfidf_transformer = TfidfTransformer()\n",
    "\n",
    "X_train_tfidf = tfidf_transformer.fit_transform(dt_dev_kmeans_nAuthor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2879,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final_dt_alpha['alpha']=train_zipf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3447,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "kmeans = KMeans(n_clusters=2, init='k-means++', max_iter=300, n_init=10, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3453,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KMeans(algorithm='auto', copy_x=True, init='k-means++', max_iter=300,\n",
       "       n_clusters=2, n_init=10, n_jobs=None, precompute_distances='auto',\n",
       "       random_state=0, tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 3453,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kmeans.fit(X_train_tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3454,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predict_Zipf=kmeans.fit_predict(X_train_tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3607,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dt_dev_kmeans_label.to_csv(r'C:\\GH-Dataset\\vector_space.csv', index=None, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3575,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X=X_train_tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3597,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X=pd.DataFrame(data=X.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3598,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Y=dt_dev_kmeans_label['real_stat']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3577,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn import svm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3599,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "clf = svm.SVC()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3600,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final_dt_alpha['svm_predict']=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(0,20):\n",
    "    X_test= X.iloc[i]\n",
    "    X_test=np.array(X_test).reshape(1, -1)\n",
    "    X_train=X.drop(i,axis=0)\n",
    "    Y_test= Y.iloc[i]\n",
    "    Y_train=Y.drop(i,axis=0)\n",
    "    clf.fit(X_train,Y_train)\n",
    "    y_pred=clf.predict(X_test)\n",
    "    final_dt_alpha['svm_predict'][i]=y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mix_result =  pd.read_csv(r'C:\\GH-Dataset\\mix_cl.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn import preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = data_mix_cl.values #returns a numpy array\n",
    "min_max_scaler = preprocessing.MinMaxScaler()\n",
    "x_scaled = min_max_scaler.fit_transform(x)\n",
    "data_mix_cl = pd.DataFrame(x_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_mix_cl['alpha']=mix_result['alpha']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_mix_cl['predict_stat']=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2793,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "z_test= train_zipf['alpha'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(0,20):\n",
    "    cl_test=data_mix_cl.iloc[i]\n",
    "    cl_train=data_mix_cl.drop(i,axis=0)\n",
    "    kmeans.fit(cl_train)\n",
    "    predict_Zipf=kmeans.fit_predict(cl_train)\n",
    "    centroid=kmeans.cluster_centers_\n",
    "    cl_cnt_0=np.sqrt(np.sum((cl_test-centroid[0])**2))\n",
    "    cl_cnt_1=np.sqrt(np.sum((cl_test-centroid[1])**2))\n",
    "    if(cl_cnt_0>cl_cnt_1):\n",
    "        data_mix_cl['predict_stat'][i]=1\n",
    "    else:\n",
    "        data_mix_cl['predict_stat'][i]=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2950,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final_dt_alpha['predict_stat_alpha_rank']=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2947,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X=train_zipf_rank.iloc[:, 0:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2949,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Y=train_zipf_rank['label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(0,20):\n",
    "    z_test= train_zipf['alpha'][i]\n",
    "    z_train=train_zipf.drop(i,axis=0)\n",
    "    kmeans.fit(z_train)\n",
    "    predict_Zipf=kmeans.fit_predict(z_train)\n",
    "    centroid=kmeans.cluster_centers_\n",
    "    z_cnt_0=abs(z_test-centroid[0])\n",
    "    z_cnt_1=abs(z_test-centroid[1])\n",
    "    if(z_cnt_0>z_cnt_1):\n",
    "        final_dt_alpha['predict_stat'][i]=1\n",
    "    else:\n",
    "        final_dt_alpha['predict_stat'][i]=0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(0,20):\n",
    "    z_test= train_rank['max_rank'][i]\n",
    "    z_train=train_rank.drop(i,axis=0)\n",
    "    kmeans.fit(z_train)\n",
    "    predict_Zipf=kmeans.fit_predict(z_train)\n",
    "    centroid=kmeans.cluster_centers_\n",
    "    z_cnt_0=abs(z_test-centroid[0])\n",
    "    z_cnt_1=abs(z_test-centroid[1])\n",
    "    if(z_cnt_0>z_cnt_1):\n",
    "        final_dt_alpha['predict_stat_rank'][i]=1\n",
    "    else:\n",
    "        final_dt_alpha['predict_stat_rank'][i]=0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2951,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2952,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rf = RandomForestClassifier(n_estimators = 100, random_state = 30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(0,20):\n",
    "    X_test= X.iloc[i]\n",
    "    X_test=np.array(X_test).reshape(1, -1)\n",
    "    X_train=X.drop(i,axis=0)\n",
    "    Y_test= Y.iloc[i]\n",
    "    Y_train=Y.drop(i,axis=0)\n",
    "    rf.fit(X_train,Y_train)\n",
    "    y_pred=rf.predict(X_test)\n",
    "    final_dt_alpha['predict_stat_alpha_rank'][i]=y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2944,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#train_zipf_rank\n",
    "train_zipf_rank['label']=final_dt_alpha['real_stat']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2881,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final_dt_alpha.to_csv(r'C:\\GH-Dataset\\final_dt_alpha_predict.csv', index=None, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2904,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final_dt_alpha.to_csv(r'C:\\GH-Dataset\\final_dt_alpha_and_rank_predict.csv', index=None, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "from sklearn.cluster import KMeans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "kmeans = KMeans(n_clusters=2, init='k-means++', max_iter=300, n_init=10, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KMeans(algorithm='auto', copy_x=True, init='k-means++', max_iter=300,\n",
       "       n_clusters=2, n_init=10, n_jobs=None, precompute_distances='auto',\n",
       "       random_state=0, tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kmeans.fit(data_mix_cl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2463,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predict_Zipf=kmeans.fit_predict(train_dt_zipf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1])"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kmeans.labels_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2905,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Evaluation = pd.DataFrame(columns=['Method','Precision(Expert)','Recall(Expert)','Precision(Novice)','Recall(Novice)'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2906,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Evaluation['Method']=['Baseline(feature_base)','Baseline(#keywords)','Zipf_alpha','Zipf_alpha_rank']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3603,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#result_performance\n",
    "result_performance= pd.read_csv(r'C:\\GH-Dataset\\result_performance.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(Evaluation_Final.to_latex(index = False, multirow = True)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3462,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import jenkspy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3528,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "jenk_cl=final_dt_alpha[['Author','alpha','max_rank','status']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "jenk_cl['predict_stat_rank']=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "jenk_cl['predict_stat_alpha']=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3560,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_jenk=jenk_cl[['max_rank','status']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(0,20):\n",
    "    j_test= train_jenk['max_rank'][i]\n",
    "    j_train=train_jenk.drop(i,axis=0)\n",
    "    j_train=j_train.sort_values(by='max_rank')\n",
    "    breaks = jenkspy.jenks_breaks(j_train['max_rank'], nb_class=2)\n",
    "    j_train['cut_jenks'] = pd.cut(j_train['max_rank'],\n",
    "                        bins=breaks,\n",
    "                        labels=['Novice', 'Expert'],\n",
    "                        include_lowest=True)\n",
    "    j_gy=j_train.groupby(by='cut_jenks').mean()\n",
    "    mean_exp = j_gy.loc['Expert']['max_rank']\n",
    "    mean_nov = j_gy.loc['Novice']['max_rank']\n",
    "    jenk_cl['break']= breaks[1]\n",
    "    j_cnt_0=abs(j_test-mean_exp)\n",
    "    j_cnt_1=abs(j_test-mean_nov)\n",
    "    if(j_cnt_0>j_cnt_1):\n",
    "        jenk_cl['predict_stat_rank'][i]='Novice'\n",
    "    else:\n",
    "        jenk_cl['predict_stat_rank'][i]='Expert'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3539,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "jenk_cl.to_csv(r'C:\\GH-Dataset\\jenk_cl.csv', index=None, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "th_bk =th_dt\n",
    "th_bk['stat']=0\n",
    "for j in range(1,20):\n",
    "    i=i+1\n",
    "    for x in range(0,20):\n",
    "        th_test= th_dt['max_rank'][x]\n",
    "        th_train=th_dt.drop(x,axis=0)\n",
    "        th_train['stat']=0\n",
    "        th_train.loc[th_train['max_rank'] < i, 'stat'] = 'Novice'\n",
    "        th_train.loc[th_train['max_rank'] >= i, 'stat'] = 'Expert'\n",
    "        th_gy=th_train.groupby(by='stat').mean()\n",
    "        mean_exp = th_gy.loc['Expert']['max_rank']\n",
    "        mean_nov = th_gy.loc['Novice']['max_rank']\n",
    "        th_cnt_0=abs(th_test-mean_exp)\n",
    "        th_cnt_1=abs(th_test-mean_nov)\n",
    "        if(th_cnt_0>th_cnt_1):\n",
    "            th_bk['stat'][x]='Novice'\n",
    "        else:\n",
    "            th_bk['stat'][x]='Expert'\n",
    "        \n",
    "    num_expert=th_bk['stat'].value_counts()['Expert']\n",
    "    num_nov=th_bk['stat'].value_counts()['Novice']\n",
    "    th_result=th_result.append({'threshold':i,'num_nov':num_nov, 'num_exp':num_expert},ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "for x in range(0,20):\n",
    "    th_test= th_dt['max_rank'][x]\n",
    "    th_train=th_dt.drop(x,axis=0)\n",
    "    th_train['stat']=0\n",
    "    th_train.loc[th_train['max_rank'] < 87, 'stat'] = 'Novice'\n",
    "    th_train.loc[th_train['max_rank'] >= 87, 'stat'] = 'Expert'\n",
    "    th_gy=th_train.groupby(by='stat').mean()\n",
    "    mean_exp = th_gy.loc['Expert']['max_rank']\n",
    "    mean_nov = th_gy.loc['Novice']['max_rank']\n",
    "    th_cnt_0=abs(th_test-mean_exp)\n",
    "    th_cnt_1=abs(th_test-mean_nov)\n",
    "    if(th_cnt_0>th_cnt_1):\n",
    "        th_dt['predict_stat'][x]='Novice'\n",
    "    else:\n",
    "        th_dt['predict_stat'][x]='Expert'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf_source.to_csv(r'C:\\GH-Dataset\\GH-dataset.csv', index=None, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_c=tf_source.groupby(['Author']).size().reset_index(name='count')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_date=np.max(raw_data['Authored_Date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_com=raw_data.groupby(['Author']).size().reset_index(name='count')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "baseline_1 = pd.DataFrame(columns=['Author','numCommits','avgInterval','lastCommit','numProject','Status'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_sec=0\n",
    "for i in range(1,len(smpl_dev)):\n",
    "    date1= smpl_dev['Authored_Date'][i-1][:19]\n",
    "    date2= smpl_dev['Authored_Date'][i][:19]\n",
    "    test1=datetime.strptime(date1, '%Y-%m-%d %H:%M:%S')\n",
    "    test2=datetime.strptime(date2, '%Y-%m-%d %H:%M:%S')\n",
    "    diff=(test1 - test2).total_seconds()\n",
    "    total_sec= total_sec+abs(diff)\n",
    "avg_interval=total_sec/len(smpl_dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "baseline_1=baseline_1.append({'Author':smpl_dev['Author'][0],'numCommits':n_c['count'][0],'avgInterval':avg_interval,\n",
    "                          'lastCommit':last_com,'numProject':numpro,'Status':'Novice'},ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "kmeans = KMeans(n_clusters=2, init='k-means++', max_iter=300, n_init=10, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = baseline_1.values #returns a numpy array\n",
    "min_max_scaler = preprocessing.MinMaxScaler()\n",
    "x_scaled = min_max_scaler.fit_transform(x)\n",
    "train_data_bs = pd.DataFrame(x_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(0,20):\n",
    "    b_test= train_data_bs.iloc[i]\n",
    "    b_train=train_data_bs.drop(i,axis=0)\n",
    "    kmeans.fit(b_train)\n",
    "    predict_Zipf=kmeans.fit_predict(b_train)\n",
    "    b_centroid=kmeans.cluster_centers_\n",
    "    b_cnt_0=np.sqrt(np.sum((b_test-b_centroid[0])**2))\n",
    "    b_cnt_1=np.sqrt(np.sum((b_test-b_centroid[1])**2))\n",
    "    if(b_cnt_0>b_cnt_1):\n",
    "        final_baseline_1['predict_stat'][i]=0\n",
    "    else:\n",
    "        final_baseline_1['predict_stat'][i]=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_bs['predict_stat_alpha_e_rf_alph']=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(0,20):\n",
    "    X_test= X.iloc[i]\n",
    "    X_test=np.array(X_test).reshape(1, -1)\n",
    "    X_train=X.drop(i,axis=0)\n",
    "    Y_test= Y.iloc[i]\n",
    "    Y_train=Y.drop(i,axis=0)\n",
    "    rf.fit(X_train,Y_train)\n",
    "    y_pred=rf.predict(X_test)\n",
    "    train_data_bs['predict_stat_alpha_e_rf_alph'][i]=y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "th_bk =th_dt\n",
    "th_bk['stat']=0\n",
    "for j in range(1,10):\n",
    "    i=i+0.01\n",
    "    for x in range(0,20):\n",
    "        th_test= th_dt['alpha'][x]\n",
    "        th_train=th_dt.drop(x,axis=0)\n",
    "        th_train['stat']=0\n",
    "        th_train.loc[th_train['alpha'] < i, 'stat'] = 'Novice'\n",
    "        th_train.loc[th_train['alpha'] >= i, 'stat'] = 'Expert'\n",
    "        th_gy=th_train.groupby(by='stat').mean()\n",
    "        mean_exp = th_gy.loc['Expert']['alpha']\n",
    "        mean_nov = th_gy.loc['Novice']['alpha']\n",
    "        th_cnt_0=abs(th_test-mean_exp)\n",
    "        th_cnt_1=abs(th_test-mean_nov)\n",
    "        if(th_cnt_0>th_cnt_1):\n",
    "            th_bk['stat'][x]='Novice'\n",
    "        else:\n",
    "            th_bk['stat'][x]='Expert'\n",
    "        \n",
    "    num_expert=th_bk['stat'].value_counts()['Expert']\n",
    "    num_nov=th_bk['stat'].value_counts()['Novice']\n",
    "    th_result=th_result.append({'threshold':i,'num_nov':num_nov, 'num_exp':num_expert},ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(5,3))\n",
    "box = plt.boxplot(alpha, showfliers=False )\n",
    "for i, line in enumerate(box['medians']):\n",
    "    line.set_color('black')\n",
    "    x, y = line.get_xydata()[1]\n",
    "    text = ' μ={:.3f}\\n σ={:.3f}'.format(rm1[i], rst1[i])\n",
    "    ax.annotate(text, xy=(x, y))\n",
    "plt.xticks([1,2], ['Expert','Novice'])\n",
    "plt.savefig('alpha_real.png',dpi=900)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}