ch5_5_BP_algorithm.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ch 5.5 BP算法\n",
    "试编程实现标准BP算法和累积BP算法，在西瓜数据集3.0上分别用这两个算法训练一个单隐层网络，并进行比较。\n",
    "\n",
    "note：\n",
    "1. 要将特征归一化\n",
    "2. 标准BP参数更新频繁，为了和累积BP达到同样的误差极小点，需要进行更多次数的迭代。\n",
    "3. 但是在累积误差达到一定程度后，进行下一步下降会非常缓慢，这时候标准BP往往能获得更好的解。\n",
    "4. 权重初始化对结果有一定影响\n",
    "5. 数据量太少，就没分训练集和测试集了"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sigmoid(x):\n",
    "    return 1/(1+np.exp(-x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dataset = pd.read_csv('data/table_4_3_watermelon_3_0_num.csv')\n",
    "dataset = dataset.drop('Idx',axis=1)\n",
    "dataset = dataset.apply(lambda x:(x-np.min(x))/(np.max(x)-np.min(x)))\n",
    "dataset_array = np.array(dataset)\n",
    "dataset_array = dataset_array[np.random.permutation(len(dataset))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def predict(x_i, w_h_j, v_i_h, theta_j, gamma_h):\n",
    "    alpha_h = np.dot(x_i, v_i_h)\n",
    "    b_h = sigmoid(alpha_h - gamma_h)\n",
    "    beta_j = np.dot(b_h, w_h_j)        \n",
    "    y_j_cap = sigmoid(beta_j - theta_j)\n",
    "    print(y_j_cap, end=\" \")\n",
    "    if y_j_cap[0][0] > y_j_cap[0][1]:\n",
    "        print(0)\n",
    "        return 0\n",
    "    else:\n",
    "        print(1)\n",
    "        return 1\n",
    "def predict_batch_data(dataset, w_h_j, v_i_h, theta_j, gamma_h):\n",
    "    for i in range(len(dataset)):\n",
    "        predict(np.array(dataset.ix[i,:-1]), w_h_j, v_i_h, theta_j, gamma_h)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 标准BP算法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "-\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.7535364346128508, 0.42269067278497496, 0.40413666400573572, 0.38470656559527477, 0.36157697426902669, 0.33614983806481974, 0.31057251656156504, 0.28683578908571822, 0.26615272161399217, 0.24887668412523531, 0.23477227614114166, 0.22333779357863612, 0.21402174016295442, 0.20632514029784713, 0.19983119122531126]\n",
      "[[ 0.13816472  0.87170754]] 1\n",
      "[[ 0.25044394  0.79536738]] 1\n",
      "[[ 0.18976537  0.79615793]] 1\n",
      "[[ 0.23925211  0.74800495]] 1\n",
      "[[ 0.22220326  0.76140484]] 1\n",
      "[[ 0.37097968  0.64139554]] 1\n",
      "[[ 0.72810462  0.31474735]] 0\n",
      "[[ 0.42107003  0.5269595 ]] 1\n",
      "[[ 0.8484318   0.14708735]] 0\n",
      "[[ 0.83595101  0.20164706]] 0\n",
      "[[ 0.94148701  0.04997007]] 0\n",
      "[[ 0.87277086  0.15619514]] 0\n",
      "[[ 0.57827274  0.36337254]] 0\n",
      "[[ 0.76970149  0.26173749]] 0\n",
      "[[ 0.33893121  0.72598782]] 1\n",
      "[[ 0.89265998  0.10468433]] 0\n",
      "[[ 0.75421076  0.22448663]] 0\n",
      "Wall time: 1.11 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "w_h_j, v_i_h, theta_j, gamma_h = standard_bp(dataset, dataset_array)\n",
    "predict_batch_data(dataset, w_h_j, v_i_h, theta_j, gamma_h)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 累计BP算法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def accumulated_bp(dataset, dataset_array):    \n",
    "    eta = 0.1\n",
    "    h_num = 10\n",
    "    j_num = 2\n",
    "    i_num = dataset.shape[1] - 1\n",
    "    l_num = 2\n",
    "    w_h_j = np.random.random([h_num, j_num])\n",
    "    v_i_h = np.random.random([i_num, h_num])\n",
    "    theta_j = np.random.random([1,j_num])\n",
    "    gamma_h = np.random.random([1,h_num])\n",
    "    error_list = []\n",
    "    for i in range(150):\n",
    "        error = 0\n",
    "        delta_w_h_j = 0\n",
    "        delta_theta_j = 0\n",
    "        delta_v_i_h = 0\n",
    "        delta_gamma_h = 0\n",
    "        for k in range(len(dataset)):\n",
    "            x_i, y_i = dataset.ix[k,:-1], dataset.ix[k,-1]  # x_i：1xi的向量\n",
    "            x_i = np.array(x_i)\n",
    "            x_i = np.reshape(x_i, [1,i_num])\n",
    "            y_j = np.zeros([1,l_num])\n",
    "            y_j[0][int(y_i)] = 1\n",
    "            alpha_h = np.dot(x_i, v_i_h)\n",
    "            b_h = sigmoid(alpha_h - gamma_h)\n",
    "            beta_j = np.dot(b_h, w_h_j)        \n",
    "            # formula 5.3\n",
    "            y_j_cap = sigmoid(beta_j - theta_j)\n",
    "            # formula 5.10\n",
    "            g_j = y_j_cap * (np.ones_like(y_j_cap)-y_j_cap)*(y_j - y_j_cap)\n",
    "            # formula 5.15\n",
    "            e_h = b_h *(np.ones_like(b_h)-b_h) * np.dot(g_j, w_h_j.T)\n",
    "\n",
    "            delta_w_h_j += eta * np.dot(b_h.T, g_j)\n",
    "            delta_theta_j += -eta * g_j\n",
    "            delta_v_i_h = +eta * (np.dot(e_h.T, x_i)).T\n",
    "            delta_gamma_h += -eta * e_h\n",
    "\n",
    "        w_h_j += delta_w_h_j\n",
    "        theta_j += delta_theta_j\n",
    "        v_i_h += delta_v_i_h\n",
    "        gamma_h += delta_gamma_h\n",
    "\n",
    "        error += 0.5*np.sum((y_j_cap - y_j)*(y_j_cap - y_j))\n",
    "        if i % 10 == 0:\n",
    "            error_list.append(error)   \n",
    "    print(error_list)\n",
    "    return w_h_j, v_i_h, theta_j, gamma_h\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.45650193468164474, 0.13387507585940717, 0.18969142353746324, 0.19210329958502109, 0.19422257925262215, 0.19610958477751503, 0.19774355963108425, 0.19908951078906162, 0.20009100995384671, 0.20066019046376204, 0.20066866783588372, 0.19994675924684968, 0.19829873919824956, 0.19553536156433893, 0.19151458488428663]\n",
      "[[ 0.35357724  0.59528503]] 1\n",
      "[[ 0.49951571  0.47578905]] 0\n",
      "[[ 0.34318829  0.57619408]] 1\n",
      "[[ 0.37968128  0.56583367]] 1\n",
      "[[ 0.36358532  0.54787434]] 1\n",
      "[[ 0.458727    0.54910606]] 1\n",
      "[[ 0.61729864  0.41860546]] 0\n",
      "[[ 0.42707911  0.53146666]] 1\n",
      "[[ 0.64656268  0.37007665]] 0\n",
      "[[ 0.70938961  0.36603967]] 0\n",
      "[[ 0.78152441  0.27163788]] 0\n",
      "[[ 0.72252828  0.32594469]] 0\n",
      "[[ 0.41275006  0.54019975]] 1\n",
      "[[ 0.62889882  0.37839398]] 0\n",
      "[[ 0.53963633  0.48693182]] 0\n",
      "[[ 0.70550882  0.31667872]] 0\n",
      "[[ 0.56465731  0.42758534]] 0\n",
      "Wall time: 1.07 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "w_h_j, v_i_h, theta_j, gamma_h = accumulated_bp(dataset, dataset_array)\n",
    "predict_batch_data(dataset, w_h_j, v_i_h, theta_j, gamma_h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}