notes/03_temporal_difference/mountain_climb_qlearning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Q Learning\n",
    "\n",
    "In this notebook, we will implement Q Learning algorithm to play Hill Climbing.\n",
    "\n",
    "## Step 1: Import the libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import random\n",
    "import gym\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import namedtuple, deque"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2: Create our environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\deepanshu\\.conda\\envs\\ai\\lib\\site-packages\\gym\\logger.py:30: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
      "  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env = gym.make('MountainCar-v0')\n",
    "env._max_episode_steps = 1000\n",
    "env.seed(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3: Viewing our Enviroment\n",
    "Execute the code cell below to play Cartpole with a random policy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The size of state is:  (2,)\n",
      "No. of Actions:  3\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAEICAYAAAB/Dx7IAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAZEUlEQVR4nO3df7BcZ33f8fcHWcgE27EdXXuEJCLVEW1kJsjxHZXGaetighUnVKZTMiI/UGacEdPaKbTuDwumSOpUU5LB0F9jJiL2RBMDivjhWmVIQFFhCDOOxZWRsWRZ+AYr1kW30rXBsZ1QBcnf/rHP2sdXe+/dX2f3PGc/r5mdPfvsOWef5969n/3eZ8/ZVURgZmb5eM2wO2BmZp1xcJuZZcbBbWaWGQe3mVlmHNxmZplxcJuZZcbBbZUl6YOSfr/f67axr5D0U/3Yl1kZ5OO4bRAk/SZwJ3AN8DzwALA1Ip4bZr9akRTAmoiYbHHf14C3AucKzb8QEQ8NqHtmrritfJLuBH4H+HfAj9MIvp8E9kt67RzbXDS4Hnbsjoi4pHC5ILQr3n/LnIPbSiXpMmAH8NsR8ScR8aOIOAH8Co3w/vW03nZJn5N0v6Tngd9MbfcX9vVeSX8p6VlJ/1HSCUlvL2x/f1pelaY7Nkt6WtIzkj5U2M96SQ9Jek7StKT/OdcLSIdjDUm3S3oSeDK1/TdJJyU9L+mQpH9YWH+7pM+mMb8g6TFJb5K0VdKZtN07Cuv/uKR7U5+/J+k/S1rUa78tPw5uK9vPARcDXyg2RsSLwB8Dv1Bo3gh8Drgc+FRxfUlrgXuAXwOW0ajcly/w2D8P/F3gJuDDkn46tZ8H/jWwFPgH6f5/2eG45nIr8PeBten2N4F1wJXAp4HPSrq4sP47gT8ErgC+BXyZxt/lcuA/Ab9XWHc3jSmanwKuA94B/Faf+m0ZcXBb2ZYCz0TEuRb3Taf7mx6KiP8VES9FxA9nrfvPgf8dEd+IiL8FPgws9AbNjoj4YUQ8CjwKvAUgIg5FxJ9HxLlU/f8e8I87GNN/T9X6c5IemXXff4mI7zf7HxH3R8Sz6bHuBpbQeDFp+rOI+HL6+XwWGAM+EhE/AvYAqyRdLulq4BeBD0TEX0fEGeDjwKYO+m014Xk4K9szwFJJF7UI72Xp/qaT8+znDcX7I+JvJD27wGP/38Ly3wCXAEh6E/AxYBz4MRp/B4cW2FfRv4qIuY5gedUY0vz+b6X+B3AZr36xOl1Y/iGNF7nzhdukfr8BWAxMS2qu/5rZj2ejwRW3le0h4Czwz4qNkl5Po4I8UGier4KeBlYUtn8d8BNd9ukTwBM0jhy5DPggoPk3advLY0jz2f+Bxnz+FRFxOfBXXT7WSRo/x6URcXm6XBYR1/aj05YXB7eVKiL+isabk/9D0gZJiyWtojEtMEVjfrcdnwPeKenn0huJO+g+bC+lcUjii5L+HvAvutxPO49zDpgBLpL0YRoVd8ciYhr4CnC3pMskvUbSNZI6meKxmnBwW+ki4ndpVLUfpRGYD9OoIG+KiLNt7uMo8Ns05n2ngReAMzSq0E79W+BX0z4+CfxRF/tox5dpvAH7HeAvgf9Hb1Mb7wVeCzwO/IDGi9myHvtoGfIJOJYlSZcAz9GY7nhq2P0xGyRX3JYNSe+U9GNpfvyjwGPAieH2ymzwSgvuNJ95XNKkpLvKehwbKRuBU+myBtgU/pfRRlApUyXpbK7v0Di5YorGSQjviYjH+/5gZmYjpqyKez0wGRHfTSdL7KFRLZmZWY/KOgFnOa9+93yKxmnALS1dujRWrVpVUlfMzPJz4sQJnnnmmZaHvJYV3K0e7FVzMpK2AFsA3vjGNzIxMVFSV8zM8jM+Pj7nfWVNlUwBKwu3V9B4Q+llEbErIsYjYnxsbKykbpiZ1U9Zwf1NYI2k1ekst03AvpIey8xspJQyVRIR5yTdQePMsUXAfenMNzMz61Fpnw4YEV8CvlTW/s3MRpXPnDQzy4yD28wsMw5uM7PMOLjNzPpIEocO9et7OVrzV5eZmZVgrvC+/vrePx/KwW1mNkCtAr3TMPdUiZlZZlxxm5kNkKdKzMwqqh8BPRdPlZiZ9VmZoQ0ObjOz7Di4zcwy4+A2M8uMg9vMLDMObjOzzDi4zcwy4+A2M8uMg9vMLDMObjOzzPR0yrukE8ALwHngXESMS7oS+CNgFXAC+JWI+EFv3TQzs6Z+VNz/JCLWRcR4un0XcCAi1gAH0m0zM+uTMqZKNgK70/Ju4NYSHsPMbGT1GtwBfEXSIUlbUtvVETENkK6varWhpC2SJiRNzMzM9NgNM7PR0evHut4QEackXQXsl/REuxtGxC5gF8D4+Hi5H6VlZlYjPVXcEXEqXZ8BHgDWA6clLQNI12d67aSZmb2i6+CW9HpJlzaXgXcAR4B9wOa02mbgwV47aWZmr+hlquRq4AFJzf18OiL+RNI3gb2SbgOeBt7dezfNzKyp6+COiO8Cb2nR/ixwUy+dMjOzufnMSTOzzPjLgs3M+iRNHb98vZCI7g6oc3CbmfWg3ZBuZ9t2g9zBbWbWgV6Cul/7dnCbmc1joTDtdrqjm8dqcnCbmbUwV4j2M6jn2/f4+Pic6zm4zcySVmFdZlB3y8FtZiMvl8BucnCb2Ujr9siOYXJwm9lIyjGwmxzcZjZScg7sJge3mY2EOgR2k4PbzGqvGNo5B3aTg9vMaqtugd3kTwc0s1oq89T0YXPFbWa1U9dKu8nBbWa10gztOgZ2k4PbzGqh7lV20YJz3JLuk3RG0pFC25WS9kt6Ml1fUbhvq6RJSccl3VxWx83MmkYptKG9Nyf/ANgwq+0u4EBErAEOpNtIWgtsAq5N29wjaVHfemtmViDpVVMjoxDa0EZwR8TXge/Pat4I7E7Lu4FbC+17IuJsRDwFTALr+9RXM7OXjVqVXdTt4YBXR8Q0QLq+KrUvB04W1ptKbReQtEXShKSJmZmZLrthZqNu1EIb+n8cd6sDJ1v+VCNiV0SMR8T42NhYn7thZnU2CkeOzKfb4D4taRlAuj6T2qeAlYX1VgCnuu+emdmrjXpoQ/fBvQ/YnJY3Aw8W2jdJWiJpNbAGONhbF83MLnwjcpQteBy3pM8ANwJLJU0B24CPAHsl3QY8DbwbICKOStoLPA6cA26PiPMl9d3MRsQovxHZyoLBHRHvmeOum+ZYfyews5dOmZk1ucq+kD9kyswqz6H9aj7l3cwqyZX23Fxxm1nlOLTn5+A2s0pxaC/MwW1mleHQbo+D28wqwaHdPge3mQ2dQ7szDm4zs8w4uM1sqFxtd87BbWZD49Dujk/AMbOB82eP9MYVt5kNlEO7dw5uMxsKh3b3HNxmNjCe0+4PB7eZDYRDu38c3GZWOod2fzm4zaxUDu3+c3CbWWmKR5BY/ywY3JLuk3RG0pFC23ZJ35N0OF1uKdy3VdKkpOOSbi6r42aWD1fb/dVOxf0HwIYW7R+PiHXp8iUASWuBTcC1aZt7JC3qV2fNLB+eIinPgsEdEV8Hvt/m/jYCeyLibEQ8BUwC63von5llyKFdrl5Oeb9D0nuBCeDOiPgBsBz488I6U6ntApK2AFsKt/1LNqsBh3b5un1z8hPANcA6YBq4O7W3eiei5W8vInZFxHhEjF9//fWNjf1GhlnWHNqD0VVwR8TpiDgfES8Bn+SV6ZApYGVh1RXAqd66aGZmRV0Ft6RlhZvvAppHnOwDNklaImk1sAY42M4+m6/QrrrN8uRqe3AWnOOW9BngRmCppClgG3CjpHU0pkFOAO8DiIijkvYCjwPngNsj4ny7nYkIJHm+2ywzDu3BWjC4I+I9LZrvnWf9ncDOXjplZvnwf8mDV7kzJ4tTJn5CmFVbsdJ2tT04lQtu8L9bZjnw9MjwVDK4wW9WmpnNpbLBDQ5vs6pytT1clQ5uMzO7UOWD21W3WXUUDxpwtT08lQ9ucHibVYG/nb06sghucHibVYVDe/iyCW5weJsNi6dHqiWr4DYzswyD21W32WC52q6e7IIbHN5mg+LQrqYsgxsc3mZlc2hXV7bBbWblcUFUbVkHt6tus/7z8drVl3Vwg8PbrCwO7erKPriLHN5mvfG8dh5qEdzFJ5nD26w7Du18LBjcklZK+qqkY5KOSnp/ar9S0n5JT6brKwrbbJU0Kem4pJvLHECTn2xmNiraqbjPAXdGxE8DbwVul7QWuAs4EBFrgAPpNum+TcC1wAbgHkmLyuj8bJ7vNuuOq+28LBjcETEdEY+k5ReAY8ByYCOwO622G7g1LW8E9kTE2Yh4CpgE1ve74/P0F3B4m7XLoZ2fjua4Ja0CrgMeBq6OiGlohDtwVVptOXCysNlUapu9ry2SJiRNzMzMdN5zM+uZC5w8tR3cki4BPg98ICKen2/VFm0XvJRHxK6IGI+I8bGxsXa70RZX3WadcbWdl7aCW9JiGqH9qYj4Qmo+LWlZun8ZcCa1TwErC5uvAE71p7vtc3ibzc9TJPlq56gSAfcCxyLiY4W79gGb0/Jm4MFC+yZJSyStBtYAB/vX5c45vM1ezaGdt4vaWOcG4DeAxyQdTm0fBD4C7JV0G/A08G6AiDgqaS/wOI0jUm6PiPN973kbIuLlJ6gkP0nNcGjXwYLBHRHfoPW8NcBNc2yzE9jZQ7/6phjeZmZ1UIszJxfi+W6zBlfb9TASwQ0ObzOHdn2MTHCbjTIXLPUyUsHtqttGkT9fu35GKrjB4W2jy6FdHyMX3ODwttHhee16GsngNjPL2cgGt6tuqztX2/U1ssENDm+rL4d2vY10cIPD2+rHoV1/Ix/cZnXiAmQ0OLhx1W314OO1R4eD28wsMw7upFh1u/K23BTntV1t15+Du8BPeDPLgYN7Fs93W258FMnocXC34PC2XDi0R5ODew4Ob6s6h/boaufLgldK+qqkY5KOSnp/at8u6XuSDqfLLYVttkqalHRc0s1lDsBsFLmgGG3tfFnwOeDOiHhE0qXAIUn7030fj4iPFleWtBbYBFwLvAH4U0lvGtYXBvei+X2V/qJhqyo/L0fTghV3RExHxCNp+QXgGLB8nk02Ansi4mxEPAVMAuv70dlh8JSJVY2nSKyjOW5Jq4DrgIdT0x2Svi3pPklXpLblwMnCZlPMH/TZcHjbsDm0DToIbkmXAJ8HPhARzwOfAK4B1gHTwN3NVVtsfsGzTNIWSROSJmZmZjru+CAV/0gc3jYsDm1raiu4JS2mEdqfiogvAETE6Yg4HxEvAZ/klemQKWBlYfMVwKnZ+4yIXRExHhHjY2NjvYxhIPzHYmZV0c5RJQLuBY5FxMcK7csKq70LOJKW9wGbJC2RtBpYAxzsX5eHx/PdNiyutq2onaNKbgB+A3hM0uHU9kHgPZLW0ZgGOQG8DyAijkraCzxO44iU23M8omQuPtLEBs2hbbMtGNwR8Q1az1t/aZ5tdgI7e+iXmeH/7qw1nznZBU+Z2CD487VtLg7uLjm8bVAc2jabg7sHDm8ri+e1bT4O7j5xeFu/OLRtIQ7uHvmPy8wGzcHdB54ysX5xtW3tcHD3icPbeuXQtnY5uPvI4W3dcmhbJxzcfebwtk45tK1TDm4zs8w4uEvgqtva5WrbuuHgLonD2xbi0LZuObgHwOFtszm0rRcO7hJFhCtvu4BD23rl4B4Ah7c1ObStHxzcZgPiF27rFwf3gLjqtiZX29YrB/cAObxHl6dIrJ8c3APm8B49Dm3rt3a+5f1iSQclPSrpqKQdqf1KSfslPZmuryhss1XSpKTjkm4ucwA5c3jXn0PbytBOxX0WeFtEvAVYB2yQ9FbgLuBARKwBDqTbSFoLbAKuBTYA90haVEbnc+XDBEeDQ9vKsmBwR8OL6ebidAlgI7A7te8Gbk3LG4E9EXE2Ip4CJoH1fe11TTi868uhbWVqa45b0iJJh4EzwP6IeBi4OiKmAdL1VWn15cDJwuZTqW32PrdImpA0MTMz08sYzCrFL8RWtraCOyLOR8Q6YAWwXtKb51m91bP2grIjInZFxHhEjI+NjbXX2xpy1V0vxUrb1baVpaOjSiLiOeBrNOauT0taBpCuz6TVpoCVhc1WAKd67mmNObzNrBPtHFUyJunytPw64O3AE8A+YHNabTPwYFreB2yStETSamANcLDfHa8bh3f+PK9tg3JRG+ssA3anI0NeA+yNiC9KegjYK+k24Gng3QARcVTSXuBx4Bxwe0ScL6f79RIRSEKS//gz49C2QVowuCPi28B1LdqfBW6aY5udwM6eezeCHN55Kf6H5N+XDYrPnKwgT5vkx6Ftg+TgriiHd/U1/ytyaNugObgrzOFdXf6d2DA5uCvO4V09fiPShs3BnQGHd3U4tK0KHNyZcHgPV/NIH3Bo2/A5uDPi8B4+h7ZVgYM7Mw7vwXOlbVXj4M5QMbwd4OXx9IhVlYM7U8UgGcXwboZqWS9ePiPSqqydzyqxippdedc5YBYK536O31W2VZ2Duwbq9vkmw/oPwlW25cJTJTWT+7RJ7v03GwRX3DXRrLqhv9MGZapSSHt6xHLi4K6RVkebVDGIqhjYUM2flVkrniqpoVE94qTTsTq0LVeuuGuqzkecbN++fd7bC3FgW+5ccddcFU/W6SUsW4X09u3b2w5vh7bVQTtfFnyxpIOSHpV0VNKO1L5d0vckHU6XWwrbbJU0Kem4pJvLHIAtrC5TJ51W1rM5tK0u2pkqOQu8LSJelLQY+IakP073fTwiPlpcWdJaYBNwLfAG4E8lvclfGDxcs486abZVSa9TIHOt78C2ulmw4o6GF9PNxeky37N/I7AnIs5GxFPAJLC+555az2Z/zVaVqu+5pkB6MXt6yKFtddHWHLekRZIOA2eA/RHxcLrrDknflnSfpCtS23LgZGHzqdQ2e59bJE1ImpiZmelhCNapYoCX+XkfC/Whab6A7ia8WwW2Q9vqpK3gjojzEbEOWAGsl/Rm4BPANcA6YBq4O63eKgEu+KuJiF0RMR4R42NjY1113nozO8yqVIEXtRu827Ztc2DbSOjoqJKIeA74GrAhIk6nQH8J+CSvTIdMASsLm60ATvWhr1aCZrgNuwJvx7Zt29i2bVvL+7Zv3/6quXsHttVZO0eVjEm6PC2/Dng78ISkZYXV3gUcScv7gE2SlkhaDawBDva32zYIVQzvVorTKQ5sGwXtHFWyDNgtaRGNoN8bEV+U9IeS1tGYBjkBvA8gIo5K2gs8DpwDbvcRJXlo9e06Zb6519zfjh07Otpurnlvh7aNClXhyT4+Ph4TExPD7oa10KrqLuM5M1d4N6dG5qr+q/D8NSvD+Pg4ExMTLZ/4Dm5r23xTJ/16Hu3YseOCNxnLeiyzKpsvuP1ZJda2+b6ouJugnWubVlMhDmuzVzi4rWOtQnS+4O7mTU4HtdncHNzWF/04I9NhbdYeB7f1nQPYrFz+WFczs8w4uM3MMuPgNjPLjIPbzCwzDm4zs8w4uM3MMuPgNjPLjIPbzCwzDm4zs8w4uM3MMuPgNjPLjIPbzCwzDm4zs8w4uM3MMuPgNjPLjIPbzCwzlfiyYEkzwF8Dzwy7LyVYiseVm7qOzePKy09GxFirOyoR3ACSJiJifNj96DePKz91HZvHVR+eKjEzy4yD28wsM1UK7l3D7kBJPK781HVsHldNVGaO28zM2lOlitvMzNrg4DYzy8zQg1vSBknHJU1KumvY/emUpPsknZF0pNB2paT9kp5M11cU7tuaxnpc0s3D6fXCJK2U9FVJxyQdlfT+1J712CRdLOmgpEfTuHak9qzH1SRpkaRvSfpiul2XcZ2Q9Jikw5ImUlstxtaViBjaBVgE/AXwd4DXAo8Ca4fZpy7G8I+AnwWOFNp+F7grLd8F/E5aXpvGuARYnca+aNhjmGNcy4CfTcuXAt9J/c96bICAS9LyYuBh4K25j6swvn8DfBr4Yl2ei6m/J4Cls9pqMbZuLsOuuNcDkxHx3Yj4W2APsHHIfepIRHwd+P6s5o3A7rS8G7i10L4nIs5GxFPAJI2fQeVExHREPJKWXwCOAcvJfGzR8GK6uThdgszHBSBpBfBLwO8XmrMf1zzqPLZ5DTu4lwMnC7enUlvuro6IaWgEIHBVas9yvJJWAdfRqE6zH1uaTjgMnAH2R0QtxgX8V+DfAy8V2uowLmi8uH5F0iFJW1JbXcbWsYuG/Phq0Vbn4xOzG6+kS4DPAx+IiOelVkNorNqirZJji4jzwDpJlwMPSHrzPKtnMS5JvwyciYhDkm5sZ5MWbZUbV8ENEXFK0lXAfklPzLNubmPr2LAr7ilgZeH2CuDUkPrST6clLQNI12dSe1bjlbSYRmh/KiK+kJprMTaAiHgO+BqwgfzHdQPwTyWdoDHl+DZJ95P/uACIiFPp+gzwAI2pj1qMrRvDDu5vAmskrZb0WmATsG/IfeqHfcDmtLwZeLDQvknSEkmrgTXAwSH0b0FqlNb3Asci4mOFu7Iem6SxVGkj6XXA24EnyHxcEbE1IlZExCoaf0f/JyJ+nczHBSDp9ZIubS4D7wCOUIOxdW3Y744Ct9A4YuEvgA8Nuz9d9P8zwDTwIxqv9LcBPwEcAJ5M11cW1v9QGutx4BeH3f95xvXzNP69/DZwOF1uyX1swM8A30rjOgJ8OLVnPa5ZY7yRV44qyX5cNI46ezRdjjZzog5j6/biU97NzDIz7KkSMzPrkIPbzCwzDm4zs8w4uM3MMuPgNjPLjIPbzCwzDm4zs8z8f1ztokWCW8WiAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"The size of state is: \", env.observation_space.shape)\n",
    "print(\"No. of Actions: \", env.action_space.n)\n",
    "env.reset()\n",
    "plt.figure()\n",
    "plt.imshow(env.render(mode='rgb_array'))\n",
    "env.close()\n",
    "plt.title('Original Frame')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_play():\n",
    "    env.reset()\n",
    "    while True:\n",
    "        env.render(mode='rgb_array')\n",
    "        action = env.action_space.sample()\n",
    "        state, reward, done, _ = env.step(action)\n",
    "        if done:\n",
    "            env.close()\n",
    "            break\n",
    "random_play()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 4: Creating State Preprocess Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_space = np.linspace(-1.2, 0.6, 12)\n",
    "vel_space = np.linspace(-0.07, 0.07, 20)\n",
    "\n",
    "def get_state(state):\n",
    "    pos, vel = state\n",
    "    \n",
    "    pos_bin = np.digitize(pos, pos_space)\n",
    "    vel_bin = np.digitize(vel, vel_space)\n",
    "    return (pos_bin, vel_bin)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 6: Creating out agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "GAMMA = 0.99            # discount factor\n",
    "LR = .1                 # learning rate\n",
    "\n",
    "class Agent():\n",
    "    def __init__(self, p_size = 12, v_size = 20, action_size = 3):\n",
    "        self.action_space = list(range(0, action_size))\n",
    "        \n",
    "        # Intilizing state\n",
    "        self.states = []\n",
    "        for p in range(p_size + 1):\n",
    "            for v in range(v_size + 1):\n",
    "                self.states.append((p, v))\n",
    "        \n",
    "        # Iitliazing Q\n",
    "        self.Q = {}\n",
    "        for s in self.states:\n",
    "            for a in self.action_space:\n",
    "                self.Q[s, a] = 0\n",
    "        \n",
    "                \n",
    "    def act(self, state, eps=.0):\n",
    "        \"\"\"Returns actions for given state as per action value function.\"\"\"\n",
    "        state = get_state(state)\n",
    "        values = np.array([self.Q[state, a] for a in self.action_space])\n",
    "        action = np.argmax(values)\n",
    "        g_action = np.random.choice(self.action_space) if np.random.random() < eps else action \n",
    "        return g_action\n",
    "        \n",
    "    def step(self, s, a, ns, r):\n",
    "        na = self.act(ns)\n",
    "        s = get_state(s)\n",
    "        ns = get_state(ns)\n",
    "        self.Q[s, a] = self.Q[s, a] + LR * (r + (GAMMA * self.Q[ns, na]) - self.Q[s, a]) \n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 7: Watching untrained agent play"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "agent = Agent(p_size=12, v_size=20, action_size= 3)\n",
    "\n",
    "# watch an untrained agent\n",
    "state = env.reset()\n",
    "for j in range(200):\n",
    "    action = agent.act(state, 0.5)\n",
    "    env.render()\n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "    state = next_state\n",
    "    if done:\n",
    "        break \n",
    "        \n",
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 8: Loading Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_epoch = 0\n",
    "eps_start = 1.0\n",
    "scores = []\n",
    "scores_window = deque(maxlen=100)\n",
    "\n",
    "# To Load checkpoint uncomment code\n",
    "# checkpoint = np.load('mountain_climb_qlearning.npy', allow_pickle=True)\n",
    "# agent.Q = checkpoint[()]['Q']\n",
    "# scores = checkpoint[()]['scores']\n",
    "# eps_start = checkpoint[()]['eps_start']\n",
    "# start_epoch = len(scores)\n",
    "\n",
    "# index = 1\n",
    "# for i in reversed(scores):\n",
    "#     scores_window.append(i)\n",
    "#     if index == 100:\n",
    "#         break\n",
    "#     index += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 9: Train the Agent with DQN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(n_episodes=50000, eps_end=0.01):\n",
    "    global eps_start\n",
    "    eps = eps_start\n",
    "    for i_episode in range(start_epoch+1, n_episodes+1):\n",
    "        state = env.reset()\n",
    "        score = 0\n",
    "        while True:\n",
    "            action = agent.act(state, eps)\n",
    "            next_state, reward, done, _ = env.step(action)\n",
    "            agent.step(state, action, next_state, reward)\n",
    "            score += reward\n",
    "            state = next_state\n",
    "            if done:\n",
    "                break\n",
    "        eps = eps - 2/n_episodes if eps > eps_end else eps_end # decrease epsilon\n",
    "        scores_window.append(score)       # save most recent score\n",
    "        scores.append(score)              # save most recent score\n",
    "        eps_start = eps\n",
    "        print('\\rEpisode {}\\tAverage Score: {:.2f}\\tEpsilion: {:.2f}'.format(i_episode, np.mean(scores_window), eps), end=\"\")\n",
    "        if i_episode % 1000 == 0:\n",
    "            print('\\rEpisode {}\\tAverage Score: {:.2f}\\tEpsilion: {:.2f}'.format(i_episode, np.mean(scores_window), eps))\n",
    "    \n",
    "    return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAEGCAYAAABCa2PoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXwU9f348dc7IYQAAgIBEdAAIiCoCBFvRcECasVW+xVtq7X9ys+jrfVb2y94VWuxXrVqPfHrrfWqVayIB4LiwWFAlEPAcAeQhDsQyPn5/bGzyWaz2SszOzO77+fjkUd2PzM7+5nd2XnP55jPR4wxKKWUUi2V5XYGlFJKpQcNKEoppWyhAUUppZQtNKAopZSyhQYUpZRStmjldgbc0rVrV1NQUOB2NpRSylcWLly4zRiTH2lZxgaUgoICioqK3M6GUkr5ioisb26ZVnkppZSyhQYUpZRSttCAopRSyhYaUJRSStlCA4pSSilbaEBRSillCw0oSimlbKEBRSmlWmDemu0Ul+51OxueoAFFKQcUl+6lYNJ0Zq8obfG2jDGUlVfakCvlhAlT5zH6/k9s3+4vnlnAtMWbbN3myu/LWb99n63bDKUBRSkHFK3bAcCMpVtavK3rXlnM8VNm8uzna1u8LeUfH68s47pXFtu6zTEPzOGMez+2dZuhNKAo5YA6ayLULJEWb+vtrzcDcNt/lrd4W0o5SQOKUg6os6bWFiugbNq1nxfmrnMvQ6oRYwwPzFxF6Z4Dtm2zts6wt7LGtu35kQYUpRxQXVsHwJbd+wE45a5Z3DJtGTv3VbmZrUZKyw9QWp78CXVN2V4OVNcm9VpjDMYKui1RXBo5D2ff/wkFk6azcP2OiK/7pmQ3D8z8jhF3ftRk2SVT51EwaXrE/N357rcUTJrOne9+22TZL55ZwJA/vc8tby3FGEPJzgoKJk1n1oqtEfMwe2Up67YF2jO2763k+92xv4u/fbCSrzbs5KX565mxZAvLN+9h/prtMV+XKhk72rBSibjt7WXsqqjigQnHxVy3praO263qqY9XljVadtHjX/DR70eycUcFE6bO4/WrTuLQTnlM/vc3vLxgI2v/ek59qSYepXsO8OjHq3n2i3UUTxlHq+z4rxFHTAmcTNfddS5zV2+ntPwA44f2bHb96to6xj/8Oc9ccTyvF23kvg9W1S/7f6f35euSXdx63mCe/HQNb361iQU3jqJL+1yys4Sy8krWlO3lhL5dqKqp48ibZwDwq1P7MPjQDtw6bRl7K2vIPyiXS0YcRl5ONvura1m8cRcVlTW8ftVJ9Z9LdW0dby7axKhB3Rh9/yece3QPHvnpMADWbtvHmfd9XJ+vCx+by6iB3bh6ZD8KCzoDUFdneCasPepAdS0VVbV0bteaudYJeuXWcgYe0oGnPlvLjn2VZIswdc4aAKbOWcON5wxqtI1Pv9sGwAvz1vPCvIYBeX/5bBHr7jq3yWd5xTNfAvDmNSfzo0e/qP8uqmrq+Hhl484c2/ZWsrp0L/+YVcw/ZhU3+W6C299dUc2xf/6ASeMGctUZ/QAomDSdXgfn0adruyavs5sGFKXi8OwX6wCaBJQz7p3N+u0VHNKhDd9b1Se/PKVPs9tZXRa4In3lyw1s2rWffy8q4ddn9eflBRuBwJXzMb06UrR+J4WHH0xlTV2j15/1t49ZU7aPNXeeQ1aWNLrCPuKmGfXpibrkyXkAjQJKZU0tA25+j7757Zj1+5H0vykQBE6IcFX/hHWiPeehT+vTRtz5ET075fH5pLMY9+Actu2t4uZzB/GX6Q1X90991vjEXlZeyUMffddk+30mv0vxlHGU7a3kpL/OAuCUI7oAMH3JFh6x1ovUceGjFaV8tKK0/qTb98Z3Gy0vmDQ9wicCYx/4lK9uOZs73rGn7WrL7v31ee91cF59ejCYBD3+yWru/3BVo7TCv8yMuf2SnRX89uWvALhrxgquOqMfuyuqrWX7Kdm5v0X5j4cGFKUi2LrnAHmts+nQJqdRelVNHa1bNZQC1m+vAKgPJgCfrIrdVbimNlCdEl6rUmcMz89dz5/eXhbxdWusgNT3xnd58rLCJstv+88yXpy3nkd/OoyxQ3o0+/6LN+6qf9xcl+SLn5jX6D2TsWnXfuau3s62vYGqvtBgkqirXlzEzG8bqo8+L068qmdXRWJVjmMfnJPwezQnGEyAZk/upXsONAkm8Tr17tlN0k6/t2mak7QNRakITrjzI0b9rem9BbdOWxrzteFVVss372myTvCK/rFPVje6QjbAog0748rjlc83nSDu+bnrqTOBk2+w/SbcgepaLnjk8/rnza0XGnRaIlj6aanQYJKsSG0m0Wzdk9r7f065e1bslRKwe3+1rduLRQOKUs2IdOUerCePJrzCKbQaKFxFVdMG5WmLN8d8j3hU10Ru9F6yaXej57V1LW8cd9t/PTE3rvWqwqoQvaa61t/fhQYUpSy7KqqYMn15fQ+tdLG/qpYZS7bw9w9X8c43m/m/T9c0Wn7R4w0n4407KiL2blq4Pr5Sk1sWrI3cmytUvCW/eBWXlse9bnPtNE5qruTpJG1DURnDmMB9AgeFtYsA7DlQzYSp81jxfTlH9+rU7DaqrGBTtG4HQ3p2jLhOrE5aqRxGZfnmPVFLSNC4hHLaPZHr3C987IuI6V5StG4Hz81tdrpzfvyovfuwZNNuundoY+s2I4mna3ekgBXaZpMqGlBU2li4fie5rbKaPdG/NH8DN7+1lI9vGElBSBfKb0p2cf7DDW0KG3dU1D9etGEnBV0a1i0rr6z/8Z53TORG71hVSMdPab7HzuIN9rRbQCCwxQom6SS0pJUK17/6dUreJ9i92w/EjpuL/KiwsNAUFTVt1FT+cca9s6mpNXw+6Syg4Spt3V3nsmprOa2yhPeWfU/r7CxWfF/OG4tKGvWqOmtgN2bZMHijV10zsh+Pfrza7WwoDwq/LyYRIrLQGNO0iyFaQlEumf7NFt78ahMPThhKu9zYh6Exhv3VtdQZGHnv7PpuqBAIJJeecFj98/9+riiuHkHpHEwADSYq5TSgqJT47Ltt9OvWjgPVdY3uZJ6x9HsKDz+Yw7u0re9uW7RuB9+U7KZft/Zc/vQCOrXNYdyQQ+pv/ovkn/M31D+2o3upUipxGlBUiwQHxOuY17ShO9TPnppPx7wcLjvp8Ebpyzbv5obXG+qi5984qkld+K6K6qjBRCmVmNkrSjlzYDfbt6vdhlWL3Pnutxx7+wfsCxtltbKmlhte/5rSPQfqJwnavb+6yZ3hn4Xd1/FSSElDKeWMK5790pHtaglFJeyWt5bSKlu4aHiv+rGY9lXWNGoLeW/p9/xrYQmVNXW0CRmq5MX5jbt1fhc2dWqkcZyUUv6gAUUlLDiS6jOfr0v4tbsqUjsUhFIqdTSgKEfNXb2dgi5t3c6GUioFNKAoWzR3N9O2vZVs25vaAfaUUu7wXKO8iNwrIitE5BsReVNEOoUsmywixSKyUkTGhKQPF5El1rKHJJEZipRSStnCcwEF+BAYYow5BlgFTAYQkaOACcBgYCzwqIhkW695DJgI9Lf+xqY605lu1dZylpTsjr2iUipteS6gGGM+MMYE+6DOA3pZj8cDrxhjKo0xa4FiYISI9AA6GGPmmsA4Ms8DF6Q84xnu508t4IcPf2bLPOFKKX/yXEAJ80tghvW4JxB6d1uJldbTehye3oSITBSRIhEpKisri7SKaqFken4ppdKDKwFFRGaKyNIIf+ND1rkJqAFeCiZF2JSJkt400ZipxphCY0xhfn5+S3dDRRAc9uS6Vxa7nBOlVKq50svLGDM62nIRuRw4DxhlGupQSoDeIav1AjZb6b0ipCullEohz1V5ichY4H+B840xFSGL3gYmiEiuiPQh0Pi+wBizBSgXkROt3l2XAdNSnnGllMpwXrwP5WEgF/jQ6v07zxhzlTFmmYi8BiwnUBV2rTEmOCH31cCzQB6BNpcZTbaqUuKL1du57/2VbmdDKeUCzwUUY8wRUZZNAaZESC8ChjiZr0xnjKHOQHZW7Ft8Hp5dnIIcKaW8xnNVXip1KmtqKd3TeL7qhz76jonPN8xkeaC6ljveWc59H6yk343vsnD9jlRnUynlE54roSjn/P61rxk35BBGH9UdgGtfWsTMb0sbTQd6/4erGr3m+bnr6kcUBrjwsdTO262U8g8NKBnkjUUlvLGopD6AzPw2MAVuyc4Klm7azVUvLqpfd+byrUz9dA1LN+nd70qp+GhAUZx69+wmaf8dUu2llEovJ/Xt4sh2tQ3FxypramOvpJRSYbIcOvNrQPGpVVvLGXDze7zzjd7DqZRKTJZDA7JrQPGpYNvGR1Y7SKi6OsPlTy/g8+LAfO01tXX8Q6fWVUo5TNtQ0lB5ZQ2frCpj0Yad/HHMAG6ZtsztLCmlPMSpKaO0hJLmbvvP8iZp23UGRaVsc7bVDd9PnJqBUAOKz7351aaEXzP8LzMdyIlSmenJywrdzkLC4hjwIrntOrNZlUq7KqrczoJSyke0UV41q7au+VkSm1v25TodQkWpTOVQPNGAkg6aCyflB2qaWdIwEZZSStlFA0oaqNN53JVSHqABJR1oPFFKJUTbUFQzojShKKVUE9rLSzXLaBFFKZUA7eWlmrVg7Q5WbS1P6DVPfLLGodwopbzOqV5eOvRKGrjulcUAjSbKUkqp5mgJJYOt+H4Pm3ftdzsbSqk0oSWUDDb2gU+B+Esgf3j9ayezo5TyOR0cUsXtg+V606JSqnnay0vF5anP1rqdBaWUx+lowwqAOavK6Dt5Onv2V0dcfsc7TYerV0qpUFrlpQB4eHYxdQa+3dK0m7DRIViUUnHQwSFVgBUzIh0QVbV1qc2LUsqXtNuwAhruinfqCkNltj+MGeB2FlQKaBuKUspxHfNy3M6C751yRBe3sxCTllAUANpMopS3TRo7yO0sxKRtKAoIHam+6RGxba9OBayU21q38v5pVXt5qUYiHQ+n3DUr9RlRaUXb5jKD3tioAO0arJylh1dm0CqvDHWgurbR8+DvXS8k1UXDe7mdhbit/es5rPrLOLezkRJ+KOWJztiooOEK0g8HrbLf4z8bVv+4T9d2LuYkMSLii7aFTJFxVV4icoOIGBHpGpI2WUSKRWSliIwJSR8uIkusZQ+JUy1OHuLUFYZSKv1lVKO8iPQGzgY2hKQdBUwABgNjgUdFJNta/BgwEehv/Y1NaYZT5EB1LYs37gK0hKIgV6/4VZIy7T6UvwN/hEaTpY8HXjHGVBpj1gLFwAgR6QF0MMbMNYEW6+eBC1Ke4xT4pmR3/WONJyo3Jzv2SkpFkDGN8iJyPrDJGBM+S1RPYGPI8xIrraf1ODw90rYnikiRiBSVlZXZmOvU2L63sv7xc3PXu5gTlSna5HjuFOEL9/3kWLezEFVaDb0iIjNFZGmEv/HATcCtkV4WIc1ESW+aaMxUY0yhMaYwPz8/+R2wydY9B3j1yw1R1ykrbwgiV7+0yOksKdXIV7f8wO0s+JLXe+Ad1qWtI9t1ZQpgY8zoSOkicjTQB/jaajTqBSwSkREESh69Q1bvBWy20ntFSPe8K575kuVb9nDmwG50O6hNxHUmvrAwxblSfjN6UHdmfuvMLJ15rbVaLR0N6dnRke16qjxrjFlijOlmjCkwxhQQCBbDjDHfA28DE0QkV0T6EGh8X2CM2QKUi8iJVu+uy4Bpbu1DIrbvC5Q+6qKMOh9azaVUJJ3b6YCOfuN2hwqnqrxcKaEkwxizTEReA5YDNcC1xpjgXX9XA88CecAM60+pjKBdyP0nXQck8HRAsUopoc+nAFMirFcEDElRtmxn0vbwUqnQKlsDipfE9W2k6U/eU1VemSbaleXGHRVs3FGRwtwo38mggbecurPbDwoPP9jtLMRNA4pHnXbPbE67Z7bb2VA+kO43uV4zsh+PXDos9opp6l9Xn2z7NjPqTnnVIHOuQZWK7I9jB3Jopzy3s2EvBy8Cfjuqv3Mbj0EDigdkUM2FUs3y02CXqXZS3/inFT62V+wuwWl1Y6MKSPeqCqUS0SGB+ezzMmzYGb+M1OyPXCqlVIgrT+/rdhZUBBpQPGTl9+VU1za+y1ELMUo1NWpgN7ezkFJ212ZkzOCQmcgA67fvY8wDc/jruysaLSst1zvlVWRp1/RmDEU3RxyVKf0k+OX5pZ1VA4qLQi8Stu2tAuCrjTvdyYzyhRMjNM6my53yBujaPtftbHiST+KJBhSl/CQ4WGPrbP3ppkrndq0TWt/tzjbxlGacugjx9NArmeLtxZvJ0eEzlFLN8MvZQQNKCu2uqGZHRVV9f/vg3ap3v9fQbmIM1NRGGX5YKZVS6TjsizbKp4FxD87hzPs+jrrO4o27OOImHSxZucPN6hovNDwf1rnpxFP/vPJEF3LiTxpQUmjz7gONnu85UO1STpRSkRwcob3kyO4HceYA92d49YO4A4qInCoiV1iP861JrlSSikv3Un6gxu1sKB/zwhW9W+woSbVN09ko3Tws4gooIvIn4H+ByVZSDvCiU5nKBMWle93OgkqBiwt7x17JMqKgc1LvEXpy/fmJhye1jUzUM2zAyfOPPZRfnOzNz8/uIDGoRwebtxgQbwnlR8D5wD4AY8xm4CBHcqRUGrn7omPiXteOq+6BPbz3s2zlk1bthy45joNyo48ndlaa3KGf7dB3Em9AqTLGGKxAKSI6LKhSNvNbDdbjPxvO+cceGnO9hTefHdf2/DBz6fihsffXCcYn9ZvxBpTXROQJoJOIXAnMBJ50LltKKa8bO+QQJsYxSGPHtvGPIuxXH1x/Ol3qG/T9USJzQlz3oRhj7hORs4E9wADgVmPMh47mLM25fTetylzdO+SydY/3xojzwkV4c1mIlbUjux9Ex7Y5bN9XZXeWEuZmaSZmQBGRbOB9Y8xoQIOIUk5J0XngoDY5ngwoqZbMx+3U1LnpImaVlzGmFqgQkdjTgCmlfO/60Ue68r7JXlgfX3CwvRlJgW4dcjmkQ5u0G3Ip3jaUA8ASEXlKRB4K/jmZMaUyToLnlkhVG62zs7jhB0dyQZTG41hvc91o9+Yk9zq7Tv/ZWcK8G0fx3ZRzbNqiN8Q7ltd060/ZJL2uS5Sbwo+lX5/Vn0dmF7uSl1Tx+5D9iZbG4l1/QHd3u43HVUIxxjwHvAwstP7+aaWpJGzdc4CJLyx0OxtKtZgT7b8z/+d01/PgV9N+fYqr7x9XCUVERgLPAesIXBD1FpHLjTFznMta+vpqwy63s6BUizjZNn1Et8Susu2MJ36536M5bXKyXb2bJ94qr78BPzDGrAQQkSMJlFiGO5WxdLa3UsfwUi2Tm5N+47r6+1TuLD/c9AnxN8rnBIMJgDFmFYHxvFQS/H4VpNyT2yqbSeMG8uY1J7udFRUugZ+1XwJEouINKEVWD6+R1t+TBNpSVDNeXrCBgknTKS0/0GSZllBUS1x1Rr9mq4XiuU9Cb6VIXjxhIJM/33gDytXAMuC3wHXAcuAqpzKVDv61sASADdsrmiz72werUp0dlYYindxSdS7Lsea0d+tK2+9X+M31Ujv36B4pzom94g0orYAHjTE/Nsb8CHgISM/JBFJASyjKTvGUSuIZxDER0661vzdRslXBTlQhj/TYhFqd2jad+OuUI7pEXNfNGvV4A8pHQOjkAXkEBohUStnFwRPBAxcPtXV7Ts2nkQxbe3nZuK3o75PYO935o6ObpJ13jDsjH0cTb0BpY4ypnxHKetx08mXVhL8L5ipdZIXMf9Epr+nVrt/YcWNjtC14rRmkY17TPlBeyyPEH1D2iciw4BMRKQT2O5Ol9ODFL1spgPyDcrn2zH5uZyNtZXInzngDyu+A10XkUxGZA7wC/NqpTInIb0RkpYgsE5F7QtIni0ixtWxMSPpwEVliLXtIdEhQ5UMj+iQ3BXAy+nZtb8t2/D4Eiq1s+Cj83tkgakARkeNF5BBjzJfAQOBVoAZ4D1jrRIZE5ExgPHCMMWYwcJ+VfhQwARgMjAUetYbWB3gMmAj0t/7GOpG3ZGTy1YqK389PPJw2KbpZcVCUaYLb58Z7r3OAnSfATPqtOLuv7n2QsY7gJ4DgjDEnATcCjwA7gakO5elq4C5jTCWAMabUSh8PvGKMqTTGrAWKgREi0gPoYIyZa01T/DxwgUN5i5uWkVSiEi1YJ3tSumbkERFPOXeMH8wPPdjQG0sqApEd73FY5/Rvdo4VULKNMTusxxcDU40xbxhjbgGOcChPRwKnich8EflERI630nsCG0PWK7HSelqPw9ObEJGJIlIkIkVlZWUOZF0p7wttoA/185MKml2WqFEDu9mynVSLFTiifjpRXvuLkwsyYnSDmAFFRIJl4FHArJBliZWNQ4jITBFZGuFvvLXdg4ETgT8QmM9eiPxdmijpTRONmWqMKTTGFObne6ufuVJ28ErJ+JoUN/o//YtCW7fXkmbYSC8d2rsTXdrnRl0nUV6sIYwVFF4GPhGRbQR6dX0KICJHALuTfVNrOuGIRORq4N9W9dUCEakDuhIoefQOWbUXsNlK7xUh3RN03C7lFK8ED7tEao959opABUW0ffXjL6y504LfTxdRSyjGmCnA74FngVNNw9kxC/iNQ3l6CzgL6kc1bg1sA94GJohIroj0IdD4vsAYswUoF5ETrZLMZcA0h/IWN+39orzM6aPTrhPjyAHeqjrz0vk+me/wpnMGsfDmZq/nWyxmtZUxZl6ENCcHo3oaeFpElhLoEHC5FciWichrBMYRqwGutea7h0BD/rME7uCfYf0ppRzm5ytqL2bdjs8z2jZysqVR1Zvdkm4HcYoxpgr4WTPLpgBTIqQXAUMczlpSgt/tM5870staqYzUqArMmIROxHP+cCZ1xvDfzxfZnq/mhFfn+TkQR5N+s/R4RVh59Pb/LHcnH8pxE47vHXulOCTaJpKuJyWnHdalLQVd27mdjbSkAUUpF/XN1xNb0OhB3d3OAiMH5HPuMT249byj3M5KTCf36+p2FprQgKKUiy4/qcDx9+jZKS/2Si5bdMvZ3PCDAa69/6M/DQxV2CYnm0cuHUbPg73/meW0ChRpD+nQxuWcNNCA4jCtlkh//fJbPi6Wk12A7/3JMY5tu7nDO9HDvnO71knfVGnHT+wcGya2Cs9H9w7ONX5H082l9wUNKI7RTsOZ41en9uHViSe6nQ0gcmCK1IXd6eucTL6QCn7a828czQVDGw9lM36ofUPbBD/j8O98+OGdbWvXS5QGFKVaKCtLOKFv5NnzEqH3LiXHT8ErVrVeIoNtBteMdNQcXxB55GqnPyoNKA7z+3DUSqmW81PQawkNKA5Jt2ExlPPsPGZ0yB9vyLSpmTSgKOVxM//njIjpGjL8J93jiwYUldEuGeFO42W4aAWKgi7R59Fw8yTllZJQS7Jx4bBesVeqfx9v7K9XaUBxmh5/Kgq7g4FXGvb9dOK96oy+bmchbWhAcYhXftjKPp/+8Uy3s6AiGDvkEMffw67w2CorcMrtmJdj0xYT4/RZSQOKUsrXrhmZusm8WtrIfkjHNvx5/GCe/sXxEZc7XbBzutzoudGG041/Cv4qllbZyZ1MWmdnUVVbF3M9LdMmxys9qaJV84UuuizKcDt+P19oCcUhHjnGlY16dExyfKdosw3afAaJ97iz6/D0+wnQCaEBzsnTQLQA5tb5RwOKUh4gkszw9fGdziNtVwNB+kikhKZtKEqpqLzYAUQDlvO82JFOA4rDvPilK6WUEzSgOETbUJTyNj/dK+MXGlCUUgm596LY86vYfa528tzv5MVfotl2OsYd3aujo9vXgOIwHW3Y27x0kerFtpBIflLYMFyNVz4/p7JhV7DxSo1Fp7atHd2+BhSH+OXkoJSKX7LVZF4JvE7TgKIymleuHFV6yrTDSwOKUj7nxaCYiVW9mbfHTWlAcVimFHX9y/mzsRvne68MR5KJUv3JR/qq2+RkR1zX6fORBhSH6O/ZL9yN+E51XdUusZlt7OBDuPGcgSl/Xw0oSnmA4NxFSKTN2hdw3Alc6XvBZs/nmZUlTDw90ijMzn5fGlAcpteJSqlMoQFFZTStGVLRBAtCLS3RZcphpgFFKZ/ydDD0ct5aIOHODk7ehe/Bz1gDilI+l7bNCQlIl04IyeyGl9qTdMZGBwy/40O276sC0udAV+nFrm7FenirUFpCcUAwmKj09cKvRridBeUxdgRXv8dnDSgqoyV7oX5a/3xb38OuE0mm3NDoVskonhqHVH0FyYxGkHE3NorIUBGZJyKLRaRIREaELJssIsUislJExoSkDxeRJdayhyRTflXK9xrNP27DYasHfubx0kC0ngsowD3A7caYocCt1nNE5ChgAjAYGAs8KiLB8QUeAyYC/a2/sanOdHP8XoRNd05esXVtnwt46wefKk4f93rJ6E1eDCgG6GA97ghsth6PB14xxlQaY9YCxcAIEekBdDDGzDWB8ujzwAWpzrRS4bLj+HWlotOGGwX2dGmst6/zQpp8IDF4sZfX74D3ReQ+AgHvZCu9JzAvZL0SK63aehyerlRMXrnSFRKvrkq0Dn3ataeQY0U5u05wB7dzdsImv8u00qkrJRQRmSkiSyP8jQeuBq43xvQGrgeeCr4swqZMlPRI7zvRapcpKisrs2NXYjNQXVuXmvdSGSneoHhs704cdWiH2CsmoF9+e/59zcmxV1QZwZUSijFmdHPLROR54Drr6evA/1mPS4DeIav2IlAdVmI9Dk+P9L5TgakAhYWFKSuDvrJgQ6reSqUREX9UHQ077GC3s5AUL360fq8a82IbymbgDOvxWcB31uO3gQkikisifQg0vi8wxmwBykXkRKt312XAtFRnOpoD1VpC8apBPey9YlfuaNlpOHOqpZwOV15sQ7kSeFBEWgEHCPTewhizTEReA5YDNcC1xpha6zVXA88CecAM688zvFJPr5oabHMVUEvocaL8znMBxRjzGTC8mWVTgCkR0ouAIQ5nLSmZOBWqUpl43Kd6n4O1Y166EPFilZdSaSWVP3gvnVzc9IcxA2zZTjIfZ6SeXU6EmuA2E8ljxt0pn470xn3lhGgnB5+37bbYRcN7xV4pBTLtp9gLLJIAAA92SURBVK8BxWGZ/sP2Pv//4jPxgiXT7u/wCw0oKXDHO8vdzoLyIT1lZh6/X39qQFHKRaEnEKcCSAYWYHzL7zUaGlCU8gARSauqK7+fGNOV0z3RNKA4TH9Yyu9GD+pGXk527BVVxtOAopSK6v8uP55v74hvRojjCwLDsKT6OsruCze9EEyOBhSlfMqL57w2aVaSSaQWMmoQ8uKX5QANKA7LkOPIt7q29//w6+nT8pIeQoOQk9+NFweS1ICiMtrhXdo5/h7JnlSSaaTX+zNSo1PbHEe2m0yI8FJnDg0oNpq7ejt1dd67alD+4KHzgufpZ+VNGlBs8smqMi55ch5PzFnjdlZUGC+NKJwpvFgdk26S+YR1LC+f+H73fgDWlO1tlK4/LO+aNG6g21lIO16qfskUXvrENaDYJFh3reHDH/rmt+OqM/q5nY16Tl54eOmEYxe9TvMmDSh2sX61eqCrcKm8ak/0rY47rJMzGfG5WHeUJ/qNZsr8MBpQbBI8wMIPnMw4jFRLiTTfP6u5Y6glFy/Bl/bt2j75jWSAlvaay7QaQA0oNqm/CtUI4jmRftR++J3HnccW7Ez6nPCS+OF58Lfq9zZXDSg2aSihKBXQoU2C9yqkz9ndV78Duz72VMcCL8YeDSg2qS+gePFbVowckJ/S97v9/ME898sRQGpLQ25f2KRPSPSRBD70QzvlOZcPoJWjW88QM5ZsYVdFNdD0h6zxxRtaZaX22unykwtS+n7J8uvxmerCnF8/p3Ad85y5wz9IA0oLbdtbydUvLap/ni4HnkpvWpJQTtAqrxaqrq1r9FzjifcIQrcOuW5nI7P4/IegF4bJ0YDSQllhZe+mbSh6ZHrBLece5XYWktbcyc2L9zakUb+CAJ822LtFA0oLhR9vGXLc+IoI5LXODkvz/pkv3izGuleie4c2NuRGJSPR+1j8Hni0DaWF/HBiUv7gxJF04bBe9O7c1oEtR5eK0tPLV55Iyc6KiMsy41fpveijAaWFmsQT733HysOcviI9tFNqSyepPJGf1K8L0CWF79jASxeS3smJVnm1WJM2lLCIsrpsXyqzo0Kc1Nedk02yUnmOSrfrHr9XFaULDSgtlBV2Egg/sO99f2XqMqMaObV/V7ez4HkeutBWCfJiENWA0kLhjW5e/JKVy1J40s6UAOHl/Ux13rxU/aYBxWZe7MqZKZbePqbR83QZBqe5YypNdi8jZMpXpQGlpWJUeanUaZ8buY9JpOu30CEojuzenmvP9M5kW0Hxdjn10AVqvXT5HbT0o030u/H7Ban28rKZvw+HzPHYT4fVP/7g+jMAeGT2areyA7R87g0v8FL1i0o9LaHYLF2uzNJdN73ZTynbaUCxnUYUpVqqR0cN+H7kSkARkZ+IyDIRqRORwrBlk0WkWERWisiYkPThIrLEWvaQWGVrEckVkVet9PkiUpDavWlMSyhKtdzsG0Y26WShvM+tEspS4MfAnNBEETkKmAAMBsYCj4pIcBCmx4CJQH/rb6yV/itgpzHmCODvwN2O5z6K7fuq3Hx7FUVwwivlHY/9dBj3/9exTdLb5GQ328lCeZcr35gx5luI2IA3HnjFGFMJrBWRYmCEiKwDOhhj5lqvex64AJhhveY26/X/Ah4WETEO9Rl97cuNPPnpmvrntWFvs3jjLifeViWhVXbgeik3J3BN0jPFw5AEtW2dTfmBmojLcrKl/n+r7Ma/h7zW2VTtr4v0MgBaW/uXE3J3bXDkhhxrWXMTi7XKaryeXdrkZDXKRyzjju4R13rZYXcQh3dgCF8eT/+G4HERbdXgoKK5rbIavU+bVk0/t+C+h57XWoe9LpbWCXwfwfcJvq8XeO0SoCcwL+R5iZVWbT0OTw++ZiOAMaZGRHYTGOBnW/jGRWQigVIOhx12WFIZ7NQ2h/7d2zdKW1O2j67tW7NtbxUn9OnM/LU7ktq2auzQjm3YvPtA/fOHLz2O6d9s4bPibQzt3Yk2OdmMHtSN/INyqaoJnHjHDTmEGUu/58fDenLlaX2pqKrl0hGB77pffnuG9u7EecdEPok9cPFQZn67FYBLTziMic8v5MEJQ/nVc0Xcfv5gAO658Bj65rcDYOrPhzc6efz1x0cz+d9LuOOCIY22+8rEk5ixdAs9O+XRpV0uyzbv5ok5a5h4el/+6/jelOzcz29G9Se3VRYrtpTzxeptXDS8FyMHdOOjb7eS2yqbN64+me+2ljfa7s9POpxteyu5amQ/yvZW8fKCDYwa1B2AK04pYFdFFRNP7xtxX3947KGs3FrONSOP4Iwj8+sDW9A7vzmVhet3Nkp773en8Xnx9ojbe+6XIyg/UM3J/bpS0KUdpx+Zz7RrT2HJpt2N1vvDmAF8sqqMW887ikUbdkbcViSDD+3Ab0f1Z+SAfOasKuPIsN/gqEHd6dKuNdeeeQT7q2vpdlAb/jx+MMMOO5j5a3dwyhFd2FVRzaad++tf8+Rlw3lj4Sb6dG3X5P3a5GQzedzA+s/z7ouO4ZnP13LOkB78YUwF5x7dg9krSzmhT8PwPs9eMYK3vtrEoSHtPzeMGUBe62x+dFzPJu8RdNsPj6KwoDPvLf2ey08uYMOOCr7bWk5e62y6tGuYx+c/vz6VxRt30qdre7bvq6RffjuuH30kFxX2YtaKUo7t1bHJtv9ywRC2763i7zNXcfO5g+L4pFtGnLr5S0RmAodEWHSTMWaatc7HwA3GmCLr+SPAXGPMi9bzp4B3gQ3AX40xo63004A/GmN+KCLLgDHGmBJr2WpghDEm8pFvKSwsNEVFRTbsqVJKZQ4RWWiMKYy0zLESSvDkn6ASoHfI817AZiu9V4T00NeUiEgroCOgRQSllEox71S+BbwNTLB6bvUh0Pi+wBizBSgXkROt3l2XAdNCXnO59fgiYJZT7SdKKaWa50obioj8CPgHkA9MF5HFxpgxxphlIvIasByoAa41xtRaL7saeBbII9AYP8NKfwp4wWrA30Ggl5hSSqkUc6wNxeu0DUUppRIXrQ3Fa1VeSimlfEoDilJKKVtoQFFKKWULDShKKaVskbGN8iJSBqxP8uVdiXAnfprTfc4Mus+ZoSX7fLgxJj/SgowNKC0hIkXN9XJIV7rPmUH3OTM4tc9a5aWUUsoWGlCUUkrZQgNKcqa6nQEX6D5nBt3nzODIPmsbilJKKVtoCUUppZQtNKAopZSyhQaUBInIWBFZKSLFIjLJ7fwkQkSeFpFSEVkaktZZRD4Uke+s/weHLJts7edKERkTkj5cRJZYyx6yphTAmnbgVSt9vogUpHL/IhGR3iIyW0S+FZFlInKdlZ62+y0ibURkgYh8be3z7VZ62u6zladsEflKRN6xnqf1/gKIyDorv4tFJDhRoXv7bYzRvzj/gGxgNdAXaA18DRzldr4SyP/pwDBgaUjaPcAk6/Ek4G7r8VHW/uUCfaz9zraWLQBOIjAd9wxgnJV+DfC49XgC8KoH9rkHMMx6fBCwytq3tN1vK3/trcc5wHzgxHTeZysf/wP8E3gnE45tKy/rgK5haa7tt+sfiJ/+rA/8/ZDnk4HJbucrwX0ooHFAWQn0sB73AFZG2jfgfWv/ewArQtIvAZ4IXcd63IrAnbji9j6H7f804OxM2W+gLbAIOCGd95nALK4fAWfREFDSdn9D8riOpgHFtf3WKq/E9AQ2hjwvsdL8rLsJzIiJ9b+bld7cvva0HoenN3qNMaYG2A10cSznCbKK68cRuGJP6/22qn8WA6XAh8aYdN/nB4A/AnUhaem8v0EG+EBEForIRCvNtf12ZcZGH5MIaena77q5fY32GXj28xGR9sAbwO+MMXusKuKIq0ZI891+m8BMp0NFpBPwpogMibK6r/dZRM4DSo0xC0VkZDwviZDmm/0Nc4oxZrOIdAM+FJEVUdZ1fL+1hJKYEqB3yPNewGaX8mKXrSLSA8D6X2qlN7evJdbj8PRGrxGRVkBHAtMyu0pEcggEk5eMMf+2ktN+vwGMMbuAj4GxpO8+nwKcLyLrgFeAs0TkRdJ3f+sZYzZb/0uBN4ERuLjfGlAS8yXQX0T6iEhrAo1Ub7ucp5Z6G7jcenw5gTaGYPoEq5dHH6A/sMAqQpeLyIlWT5DLwl4T3NZFwCxjVb66xcrjU8C3xpj7Qxal7X6LSL5VMkFE8oDRwArSdJ+NMZONMb2MMQUEfpOzjDE/I033N0hE2onIQcHHwA+Apbi53243KvntDziHQE+h1cBNbucnwby/DGwBqglcefyKQH3oR8B31v/OIevfZO3nSqxeH1Z6oXXgrgYepmHEhTbA60AxgV4jfT2wz6cSKKJ/Ayy2/s5J5/0GjgG+svZ5KXCrlZ62+xyS35E0NMqn9f4S6G36tfW3LHg+cnO/degVpZRSttAqL6WUUrbQgKKUUsoWGlCUUkrZQgOKUkopW2hAUUopZQsNKEolSERqrdFdg39RR50WkatE5DIb3nediHRN4nVjROQ2ETlYRN5taT6Uao4OvaJU4vYbY4bGu7Ix5nEnMxOH04DZBEab/tzlvKg0pgFFKZtYQ3+8CpxpJV1qjCkWkduAvcaY+0Tkt8BVQA2w3BgzQUQ6A08TuFGtAphojPlGRLoQuBk1n8BNZRLyXj8DfktgGoX5wDUmMH5XaH4uJjDCbF9gPNAd2CMiJxhjznfiM1CZTau8lEpcXliV18Uhy/YYY0YQuNv4gQivnQQcZ4w5hkBgAbgd+MpKuxF43kr/E/CZMeY4AkNgHAYgIoOAiwkMDDgUqAV+Gv5GxphXaZj/5mgCd0Ifp8FEOUVLKEolLlqV18sh//8eYfk3wEsi8hbwlpV2KnAhgDFmloh0EZGOBKqofmylTxeRndb6o4DhwJfWqMl5NAwAGK4/geE0ANoaY8rj2D+lkqIBRSl7mWYeB51LIFCcD9wiIoOJPkR4pG0I8JwxZnK0jFhTwnYFWonIcqCHNUfKb4wxn0bfDaUSp1VeStnr4pD/c0MXiEgW0NsYM5vAZFCdgPbAHKwqK2s+j23GmD1h6eOA4NzgHwEXWXNgBOcQPzw8I8aYQmA6gfaTewgMHjhUg4lyipZQlEpcnnWlH/SeMSbYdThXROYTuFi7JOx12cCLVnWWAH83xuyyGu2fEZFvCDTKB4cLvx14WUQWAZ8AGwCMMctF5GYCM/VlERg9+lpgfYS8DiPQeH8NcH+E5UrZRkcbVsomVi+vQmPMNrfzopQbtMpLKaWULbSEopRSyhZaQlFKKWULDShKKaVsoQFFKaWULTSgKKWUsoUGFKWUUrb4/9jONcM83kJRAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "scores = train()\n",
    "# plot the scores\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(len(scores)), scores)\n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episode #')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Your total score is:  -264.0\n"
     ]
    }
   ],
   "source": [
    "# watch an trained agent\n",
    "state = env.reset()\n",
    "score = 0\n",
    "while True:\n",
    "    env.render()\n",
    "    action = agent.act(state)\n",
    "    state, reward, done, _ = env.step(action)\n",
    "    score += reward\n",
    "    if done:\n",
    "        print(\"Your total score is: \", score)\n",
    "        break \n",
    "        \n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {\n",
    "    \"Q\": agent.Q,\n",
    "    \"scores\": scores,\n",
    "    \"eps_start\": eps_start\n",
    "}\n",
    "np.save(\"mountain_climb_qlearning.npy\", data)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}