muzero_tictactoe.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this is MuZero on Tic Tac Toe!\n",
    "# https://arxiv.org/abs/1911.08265\n",
    "# The MCTS works"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%pylab inline\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import gym\n",
    "from tqdm import tqdm, trange\n",
    "import os,sys\n",
    "sys.path.append(os.getcwd())\n",
    "np.set_printoptions(suppress=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]\n",
      "([0, 0, 0, 0, 1, 0, 0, 0, 0, 0, -1], 0, False, None)\n",
      "([-1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1], 0, False, None)\n",
      "([-1, 0, 0, 1, 1, 0, 0, 0, 0, 0, -1], 0, False, None)\n",
      "([-1, -1, 0, 1, 1, 0, 0, 0, 0, 0, 1], 0, False, None)\n",
      "([-1, -1, 0, 1, 1, 0, 1, 0, 0, 0, -1], 0, False, None)\n",
      "([-1, -1, -1, 1, 1, 0, 1, 0, 0, 1, 1], 1, True, None)\n",
      "1 -1\n"
     ]
    }
   ],
   "source": [
    "# The Tic Tac Toe game\n",
    "\n",
    "class TicTacToe():\n",
    "  def __init__(self, state=None):\n",
    "    self.reset()\n",
    "    if state is not None:\n",
    "      self.state = state\n",
    "\n",
    "  def reset(self):\n",
    "    self.done = False\n",
    "    self.state = [0]*11\n",
    "    self.state[-1] = 1\n",
    "    return self.state\n",
    "   \n",
    "  class observation_space():\n",
    "    shape = (11,)\n",
    "\n",
    "  class action_space():\n",
    "    n = 9\n",
    "\n",
    "  def render(self):\n",
    "    print(\"turn %d\" % self.state[-1])\n",
    "    print(np.array(self.state[0:9]).reshape(3,3))\n",
    "    \n",
    "  def value(self, s):\n",
    "    ret = 0\n",
    "    for turn in [-1, 1]:\n",
    "      for i in range(3):\n",
    "        if all([x==turn for x in s[3*i:3*i+3]]):\n",
    "          ret = turn\n",
    "        if all([x==turn for x in [s[i], s[3+i], s[6+i]]]):\n",
    "          ret = turn\n",
    "      if all([x==turn for x in [s[0], s[4], s[8]]]):\n",
    "        ret = turn\n",
    "      if all([x==turn for x in [s[2], s[4], s[6]]]):\n",
    "        ret = turn\n",
    "    # NOTE: this is not the value, the state may be won\n",
    "    return ret*s[-1]\n",
    "  \n",
    "  def dynamics(self, s, act):\n",
    "    rew = 0\n",
    "    s = s.copy()\n",
    "    if s[act] != 0 or s[-2] != 0:\n",
    "      # don't move in taken spots or in finished games\n",
    "      rew = -10\n",
    "    else:\n",
    "      s[act] = s[-1]\n",
    "      rew += self.value(s)\n",
    "    if s[-2] != 0:\n",
    "      rew = 0\n",
    "    else:\n",
    "      s[-2] = self.value(s)\n",
    "    s[-1] = -s[-1]\n",
    "    return rew, s\n",
    "  \n",
    "  def step(self, act):\n",
    "    rew, self.state = self.dynamics(self.state, act)\n",
    "    if rew != 0:\n",
    "      self.done = True\n",
    "    if np.all(np.array(self.state[0:9]) != 0):\n",
    "      self.done = True\n",
    "    return self.state, rew, self.done, None\n",
    "  \n",
    "# Play a quick round\n",
    "env = TicTacToe()\n",
    "print(env.reset())\n",
    "print(env.step(4))\n",
    "print(env.step(0))\n",
    "print(env.step(3))\n",
    "print(env.step(1))\n",
    "print(env.step(6))\n",
    "print(env.step(2))\n",
    "print(env.state[-1], env.value(env.state))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.         0.         0.         0.         0.         0.\n",
      " 0.00033535 0.         0.99966465]\n",
      "[0. 0. 1. 0. 0. 0. 0. 0. 0.]\n",
      "[0. 0. 1. 0. 0. 0. 0. 0. 0.]\n",
      "[0. 0. 0. 0. 0. 0. 0. 1. 0.]\n",
      "[0. 1. 0. 0. 0. 0. 0. 0. 0.]\n"
     ]
    }
   ],
   "source": [
    "# a mock representation, dynamics, and prediction function\n",
    "class MockModel():\n",
    "  def ht(self,s):\n",
    "    return s\n",
    "  def gt(self, s, a):\n",
    "    #print(s, a)\n",
    "    return env.dynamics(s,a)\n",
    "  def ft(self,s):\n",
    "    #print(s, env.value(s))\n",
    "    return np.array([1/9]*9), env.value(s)\n",
    "\n",
    "# unit tests for the MCTS!\n",
    "from muzero.mcts import mcts_search, print_tree\n",
    "mm = MockModel()\n",
    "obs = [1, -1, 1, -1, 1, -1, 0, 0, 0,  0,1]\n",
    "policy, node = mcts_search(mm, obs, 1000)\n",
    "print(policy)\n",
    "act = np.random.choice(list(range(len(policy))), p=policy)\n",
    "assert act == 8 or act == 6\n",
    "obs = [-1, -1, 0, 1, 1, 0, 1, 0, 0,  0,-1]\n",
    "policy, node = mcts_search(mm, obs, 1000)\n",
    "print(policy)\n",
    "act = np.random.choice(list(range(len(policy))), p=policy)\n",
    "assert act == 2\n",
    "obs = [1,0,0,1,-1,0,-1,0,0,   0,1]\n",
    "policy, node = mcts_search(mm, obs, 1000)\n",
    "print(policy)\n",
    "act = np.random.choice(list(range(len(policy))), p=policy)\n",
    "assert act == 2\n",
    "obs = [0,1,-1,0,1,0,0,0,0,  0,-1]\n",
    "policy, node = mcts_search(mm, obs, 1000)\n",
    "print(policy)\n",
    "act = np.random.choice(list(range(len(policy))), p=policy)\n",
    "assert act == 7\n",
    "obs = [0,0,0, 0,-1,0, 1,-1,1,  0,1]\n",
    "policy, node = mcts_search(mm, obs, 1000)\n",
    "print(policy)\n",
    "act = np.random.choice(list(range(len(policy))), p=policy)\n",
    "assert act == 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      "0\n",
      "turn -1\n",
      "[[1 0 0]\n",
      " [0 0 0]\n",
      " [0 0 0]]\n",
      "[0. 0. 0. 0. 0. 0. 1. 0. 0.]\n",
      "6\n",
      "turn 1\n",
      "[[ 1  0  0]\n",
      " [ 0  0  0]\n",
      " [-1  0  0]]\n",
      "[0. 0. 0. 0. 0. 0. 0. 1. 0.]\n",
      "7\n",
      "turn -1\n",
      "[[ 1  0  0]\n",
      " [ 0  0  0]\n",
      " [-1  1  0]]\n",
      "[0. 0. 0. 0. 0. 0. 0. 0. 1.]\n",
      "8\n",
      "turn 1\n",
      "[[ 1  0  0]\n",
      " [ 0  0  0]\n",
      " [-1  1 -1]]\n",
      "[0. 0. 0. 0. 0. 1. 0. 0. 0.]\n",
      "5\n",
      "turn -1\n",
      "[[ 1  0  0]\n",
      " [ 0  0  1]\n",
      " [-1  1 -1]]\n",
      "[0. 1. 0. 0. 0. 0. 0. 0. 0.]\n",
      "1\n",
      "turn 1\n",
      "[[ 1 -1  0]\n",
      " [ 0  0  1]\n",
      " [-1  1 -1]]\n",
      "[0. 0. 1. 0. 0. 0. 0. 0. 0.]\n",
      "2\n",
      "turn -1\n",
      "[[ 1 -1  1]\n",
      " [ 0  0  1]\n",
      " [-1  1 -1]]\n",
      "[0. 0. 0. 0. 1. 0. 0. 0. 0.]\n",
      "4\n",
      "turn 1\n",
      "[[ 1 -1  1]\n",
      " [ 0 -1  1]\n",
      " [-1  1 -1]]\n",
      "[0. 0. 0. 1. 0. 0. 0. 0. 0.]\n",
      "3\n",
      "turn -1\n",
      "[[ 1 -1  1]\n",
      " [ 1 -1  1]\n",
      " [-1  1 -1]]\n"
     ]
    }
   ],
   "source": [
    "# computer can play against itself...and tie!\n",
    "\n",
    "gg = TicTacToe()\n",
    "done = False\n",
    "while not done:\n",
    "  policy, node = mcts_search(mm, gg.state, 2000)\n",
    "  print(policy)\n",
    "  act = np.random.choice(list(range(len(policy))), p=policy)\n",
    "  print(act)\n",
    "  _, _, done, _ = gg.step(act)\n",
    "  gg.render()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "(11,) 9\n"
     ]
    }
   ],
   "source": [
    "# Now we try to learn a model, and things work less well\n",
    "\n",
    "from muzero.model import MuModel\n",
    "m = MuModel(env.observation_space.shape, env.action_space.n, s_dim=64, K=5, lr=0.001)\n",
    "print(env.observation_space.shape, env.action_space.n)\n",
    "\n",
    "from muzero.game import Game, ReplayBuffer\n",
    "from muzero.mcts import naive_search, mcts_search\n",
    "replay_buffer = ReplayBuffer(200, 16, m.K)\n",
    "rews = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def play_game(env, m):\n",
    "  import random\n",
    "  game = Game(env, discount=0.99)\n",
    "  while not game.terminal():\n",
    "    # TODO: Do we need to limit the depth of the MCTS search?\n",
    "    #policy = naive_search(m, game.observation, T=1)\n",
    "    policy, _ = mcts_search(m, game.observation, 30)\n",
    "    game.act_with_policy(policy)\n",
    "  return game"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2 -10 [4, 4] 5.3141327\n",
      "4 -10 [8, 5, 6, 5] 64.46634\n",
      "4 -10 [8, 6, 4, 6] 109.44394\n",
      "4 -10 [4, 6, 5, 6] 79.12269\n",
      "4 -10 [8, 6, 5, 6] 117.92034\n",
      "4 -10 [4, 6, 8, 6] 137.791\n",
      "6 -10 [0, 7, 1, 8, 3, 8] 228.86325\n",
      "3 -10 [0, 5, 5] 178.95453\n",
      "5 -10 [3, 1, 2, 6, 2] 140.30872\n",
      "3 -10 [1, 7, 7] 170.76105\n",
      "2 -10 [6, 6] 154.14026\n",
      "4 -10 [4, 6, 5, 6] 142.79263\n",
      "2 -10 [5, 5] 149.86656\n",
      "3 -10 [7, 6, 7] 592.29877\n",
      "6 -10 [5, 2, 6, 0, 7, 5] 221.93646\n",
      "2 -10 [2, 2] 167.05666\n",
      "5 1 [8, 3, 6, 0, 7] 150.44774\n",
      "4 -10 [3, 5, 2, 3] 142.98276\n",
      "4 -10 [3, 6, 5, 3] 151.6984\n",
      "4 -10 [3, 6, 0, 0] 137.26353\n",
      "2 -10 [6, 6] 127.44191\n",
      "5 1 [2, 6, 8, 3, 5] 174.89452\n",
      "4 -10 [0, 6, 3, 6] 1299.0835\n",
      "4 -10 [8, 3, 4, 8] 362.81866\n",
      "4 -10 [5, 1, 0, 1] 205.2476\n",
      "3 -10 [6, 5, 6] 162.90738\n",
      "6 -10 [5, 2, 3, 6, 8, 8] 146.33574\n",
      "4 -10 [0, 5, 6, 5] 173.46304\n",
      "4 -10 [7, 6, 2, 6] 159.79738\n",
      "4 -10 [1, 6, 4, 6] 183.58522\n"
     ]
    }
   ],
   "source": [
    "from muzero.model import reformat_batch\n",
    "import collections\n",
    "\n",
    "for j in range(30):\n",
    "  game = play_game(env, m)\n",
    "  replay_buffer.save_game(game)\n",
    "  for i in range(20):\n",
    "    m.train_on_batch(replay_buffer.sample_batch())\n",
    "  rew = sum(game.rewards)\n",
    "  rews.append(rew)\n",
    "  print(len(game.history), rew, game.history, m.losses[-1][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x152721050>]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAdLklEQVR4nO3dfaxs11nf8e9zZs6Zc33mYCfy9YtiO07AURVCGpoTCyqgEBwIVYWhatJEihSEwAjxEtpKvFmCpCoSSmlpq7agC4kE4sVNSUIioE1iiZJGIi/XiUPshAQ3OCTOi28SwLPHd/Y5M7P6x95rZu6555x52Xvfvdbev48U3XveZva6E//OmmetZ21zziEiIs21VfcFiIhItRT0IiINp6AXEWk4Bb2ISMMp6EVEGq5b9wUsuvHGG92dd95Z92WIiETloYce+rJz7vxpXw8q6O+8804uXrxY92WIiETFzD5z1tdVuhERaTgFvYhIwynoRUQaTkEvItJwCnoRkYZT0IuINJyCXkSk4RT0IhX7+8tHvOPhJ+q+jFL9n08+yd985em6L0NWpKAXqdg7H36C1z3wME8+Nar7UkrzE7//Ed70vk/XfRmyIgW9SMWeGo3zP49qvpJyTKeOwWg8G5eET0EvUrFBHoiDhgTj8LBZ42kDBb1IxZL0KP+zGcHox+HHJeFT0ItUbJhO8j+bEfR+HH5cEj4FvUjFmla68eNoyjuUNlDQi1RsPgNuRjD6mbyCPh4KepGKzWvazQjG2ZpDQ96htIGCXqRiw1nQN6Om7cdx+WjCZOpqvhpZhYJepGKDhu1SSRb6AZryLqXpFPQiFfMljqaUOhbDXUEfBwW9SIUmU8flI7942azSDTRngbnpFPQiFbpy9tuQ0s3COJqyZbTpFPQiFVqc8TalwWioGX10FPQiFfIz+u6WNaaePRiN6W4ZoBp9LCoPejN7uZl90sweM7Ofrfr5RELiSxs37fcaU+ZI0iNu2u9lf2/ImJqu0qA3sw7w34DvAZ4PvNrMnl/lc4qExJc2br5+tzFljmE64ebrdwHN6GNR9Yz+buAx59ynnXOHwAPAvRU/p0gwfBDeev0ul48mjCfTmq+ouCQdc8vXKOhjUnXQPwv47MLHn8s/N2Nm95nZRTO7eOnSpYovR+Ta8kF4cx6Mw8P4F2STdMwN123T62415l1K09W+GOucu+CcO3DOHZw/f77uyxEpla9hN2kGnIzG9Htd9ne7s65fCVvVQf8EcPvCx7flnxNpheMz+tgXL8eTKZePJuz1uuz1utGPpy2qDvoPAXeZ2XPMbAd4FfDOip9TJBjDdEyvu8UN120D8c/ofemp3+vS73VVuolEt8oHd86NzezHgXcBHeDNzrlHq3xOkZAM0jH7u1mZA+IPen/9+7tZ0Kt0E4dKgx7AOfcnwJ9U/TwiIRqm41mZw38cM3/9e/mM/otPjWq+IllF7YuxIk3mFy77edDHXtP2TV/9Xpf+bjf6dyhtoaAXqdAgn9H7oI+91OGDva/F2Kgo6EUqNEzH7DewdNPf7bLf04w+Fgp6kQol+Yx+u7NFr7sVfTD6GfzeTvbLKx1POWpAt2/TKehFKjRMx/TzHTf7DahpH991A/G/S2kDBb1IhQajrHQDWV079pp2srjrJv8F1pRTOZtMQS9SkaPJlHQ8ndXn9xrQYOQbwLY7W/MZ/WHcY2oDBb1IRYYLO1T8n7HvuvENYEBjtoy2gYJepCKLe879n7GHYjIaX/EOBeLfMtoGCnqRiviShq9l93e70Zc5hul49ovLz+xjL0e1gYJepCJJA2f0g4WgV+kmHgp6kYos7lCBPOgjn/0uzuj9uGIfUxso6EUqsrjnHLKgT8dTDsfxNhglC30BfQV9NBT0IhWZdZEemwHHXNNeXIztbBnntjsq3URAQS9SkeT49soGnEmfpPMGMGjGAnMbKOhFKnI86PcjL3X4BrD+QtDv97rqjI2Agl6kIsN0zLntDp0tA+Iv3QyPLS77v8c6njZR0ItUZHHhEualm1gbjGYNYItjasBOojZQ0ItUZDAaX1HmiH3f+fFSFGQzepVuwqegF6nI4p5zIPpjfY+f3QPZ1lEtxoZPQS9SkeR40Ee+68aXnK4q3WhGHzwFvUhFknRy5cLlTtxBf9KMPluMndR1SbIiBb1IRZL0aNYVC1mD0XU78TYYHT+7B7LSzeFkSjpW2IdMQS9SkayLtHPF5/Yi3qVy/OwegL2dbHyx/vJqCwW9SEWG6YR+b/uKz+03IOivXHfIxqfyTdgU9CIVSMcTDidT+k2a0Y+ubAADZuMbpEd1XZasQEEvUgE/w12c/fqPo91eeXhlAxgwe8eiGX3YKgt6M/v3ZvaXZvYXZvZ2M7uhqucSCc1s4XL3ytJNfzfeBqPB6MoDzWBxy6hm9CGrckb/HuAFzrkXAp8Cfq7C5xIJyryefWXppt+Lt8FomI6vWIiF+fgSzeiDVlnQO+fe7Zzz/49+P3BbVc8lEpp50B+b0UfcYHS8AQzm44t1TG1xrWr0Pwj8r5O+YGb3mdlFM7t46dKla3Q5ItXypYwmba8cjK6e0e/NZvQq3YSsUNCb2YNm9sgJ/7t34XvuB8bA7570GM65C865A+fcwfnz54tcjkgwfClj/9ji5f5ul6OJi7LBaHg4vmo8827f+MbTJt3l33I659w9Z33dzH4A+GfAdzrnXJHnEonJvIv06tKN/3qv37nq50KWjK4u3WxtWdTlqLaoctfNy4GfBr7XOfd0Vc8jEqL5TTquLt1kX49vBjw8dnaPt9frRLtltC2qrNH/V2AfeI+ZPWxmv17hc4kExZ/06Esbnp8Rx9Zg5BvAjpduQDcfiUGh0s1ZnHNfV9Vji4QuGY3Z2+mwtdBFCvHefMRfrz/bZlG/1432rlltoc5YkQoM06u7SGHeYBTbXvpZp++xBrDsc/F2+7aFgl6kAiftOYeF0k1kM3pfajptTLG9Q2kbBb1IBZYFfWyLsaed3QNx9wa0hYJepALJktJNbA1G/npPGlPMRy+3hYJepALZYuzVoXjddpw36hiMTj67B+YzerXKhEtBL1KB02b0swajaEs3Jy/GTqaOdDy91pclK1LQi1QgSa8+0tfLgr5ZpRuIb4G5TRT0IiVzzp14pK+XdZLGNaP370B86WnRvNtXQR8qBb1IydLxlPHUnTj7hWwvemwNRv6cm+MNYLDQBBbZmNpEQS9SsvnC5Wmlmw7JKL7SzfFze7xYewPaREEvUrJhuizou9GVbobp5PTx7Kp0EzoFvUjJkqVBvx1dmWOQjk88/gBUuomBgl6kZMuDvhNdKA7T8Yl76EFBHwMFvUjJZjcdOXUxNr4Go5NuOuLNu30V9KFS0IuUzJ9Mefr2yvgajJIztoue2+6wZarRh0xBL1Iyv/vktIapGBuMzmoAMzP2et2oxtM2CnqRkvkSxlkz+sXvC51z7swZPeguU6FT0IuUbJiOMYPrTrgbEyweVRxHMKbjKZMzGsDAbxmNYzxtpKAXKdkgX7g0u7qLFOaLl7GUOpaVomC+wCxhUtCLlGx4yk1HvNhm9MMlpShQ6SZ0CnqRkp12dykvtn3ny/oC/NdiO2O/TRT0IiVbZeESiOZgs2Vn94BuJxg6Bb1IyZJ0zP5ZC5eRnQ0zO7tnyWKsgj5cCnqRkp3VRQrzBqNYSh2rlG72d7NdNzF1+7aJgl6kZGfddATmDUaxzIBXCfq9Xpepg8tHcZ3K2RYKepGSDZYsxkK2VTG6oF9SuoF43qW0jYJepET+NoLLgn4vol0qyWjMlmUlp9PEtsDcNpUHvZn9GzNzZnZj1c8lUrfLRxOm7uzZL2Rf94efhc7vIjqtAQzi6w1om0qD3sxuB74L+Jsqn0ckFMkKWxH912PpjD3rQDNvdlRxJGNqm6pn9L8K/DSgpXhphVUWLv3XY5n9LltchviawNqmsqA3s3uBJ5xzH13yffeZ2UUzu3jp0qWqLkfkmlgn6GMJxSQdLy9FKeiDdvart4SZPQjccsKX7gd+nqxscybn3AXgAsDBwYFm/hI1X7pYNgOOaTF2MDq7AQziO3q5bQoFvXPunpM+b2bfADwH+Gi+gHMb8GEzu9s598UizykSMh90y4Jxf7dLcpg1GJ21yBmCYTrm1ut3z/yefd1OMGiFgv40zrmPATf5j83sceDAOfflKp5PJBTrlG6cg6cPJ0tn/3VbdkgbQK+7RXfLonmX0jbaRy9SolWO9F38egwLsssOaYN5t28M42mjazKVcM7deS2eR6RugzVKN/77bzrzO+vlbyO4bDyQbxlV0AdJM3qREiWjMZ0to9c9+z+tvZ049p0/fTjBueXvUEBn0odMQS9SIn/8wbIF1liOKh6uuOYAcXX7to2CXqREqxxoBvGcDTNYI+hj2jLaNgp6kRKtcqAZxHM2zDoz+phO5GwbBb1IiVbpIoWFs2ECD8bZ2T0rLsaGPp62UtCLlChJV9sXH8uRAav2BQD59krdeCRECnqREiWjo6UnPUI8DUbrBH1/N5vRT6c6ySQ0CnqREmXNRaffoMOL5XaCyYoNYAD9fNzaeRMeBb1IiYbphH5ve6XvjaGmverZPcBs3CrfhEdBL1KS6dStvBgL+cFmoZduRmO6KzSAweIC81HVlyVrUtCLlOTpo2wm21+hdAP54mXgZQ5/05FVTtj04040ow+Ogl6kJPPbCK5Rugl8Rr9qAxjMxx36mNpIQS9SEl+yWGUxFuI4BCwZrR70e7MZvUo3oVHQi5TElyxWWbiEOO4bOzxcY83Bz+hVugmOgl6kJGuXbiJZjF25dOMXY0ea0YdGQS9Skvme83UWYydBNxitcncpb2+2j14z+tAo6EVKMttzvuKM3nfQhrzzZp2g73U77HS2GAT+LqWNFPQiJfEli3Vm9BD2eTfJaPltBBft9TpajA2Qgl6kJL5kseriZeg3H5lOHcPDycrjgfzmI1qMDY6CXqQkg9GYnc4Wve5qM3pfugm11OFLSqsc0ub1e9vBjqfNFPQiJRmueKCZtze7+UiYM2B/XeuUbvq9TrDvUNpMQS9SknXOuYHFM+nDrGn761p3TCGvObSVgl6kJIPRmL2d9YM+1FLHYNYXsN67FAV9eBT0IiUZpuOVu2Ih/MVYX7pZtQEM8hM5Ax1PmynoRUqyzp5zWDwbJsxgnJVu1qrRh9/t20YKepGS+CN9V+UbjEI9GyaZzejX+eXV5fLRhEnA3b5tpKAXKclgzdIN+PusBroYO9psMRbCfZfSVpUGvZn9hJn9pZk9amZvrPK5ROo2TNdbjIWsfBPs9spDv71y9cXYfi/sdYe2Wu//lWsws+8A7gX+oXMuNbObqnoukbpNpo6n1+wihbAbjNZtAIPF2wmGOaa2qnJG/6PALzvnUgDn3JMVPpdIrXywrVPPzr4/3LNhkvRordk8zJurQv3l1VZVBv3zgG81sw+Y2Z+Z2UtO+iYzu8/MLprZxUuXLlV4OSLVGW4c9OGeDTNM13+Hsq/STZAKlW7M7EHglhO+dH/+2M8Evgl4CfAWM3uuc+6K5Xjn3AXgAsDBwYGW6iVKsxn92oux2zz+laeruKTCBqPxWnvoQaWbUBUKeufcPad9zcx+FHhbHuwfNLMpcCOgabs0zvymI5uUbsIMxWE6XqsrFpgtRoc6praqsnTzh8B3AJjZ84Ad4MsVPp9IbXyT0DonPULYDUbrNoDB/H65oY6praoM+jcDzzWzR4AHgNceL9uINMWmM3rfYDSeTKu4rEKSNRvAII6bqbRRZdsrnXOHwGuqenyRkGy+68bfTnDC9efC6l9MNmgA2+5s0etuaTE2MGH9P0skUrPSzbq7VAJevExG65duIBvTIMDxtJmCXqQEwwKlm8WfD8Vk6rh8NFl7PJCNKbTxtJ2CXqQESTqm191iu7Pef1Khnkm/aSnK/4wWY8OioBcpwWCDHSoQ7iFgRYJ+r6fSTWgU9CIlGK55G0Ev1JuPDDdsAINsi2lo42k7Bb1ICTZduJzN6AMrdcxvI7jZL6/Q3qG0nYJepASb7DmHcEs3m57dA1qMDZGCXqQESTpeuysWwm0w2vTsHshKN6EtLredgl6kBJvO6H2DUXBBnwf1ujdSgeyXVzqechRgt29bKehFSrDpYixkDUbBBX26WQMY6C5TIVLQi5RgMNqsdANh7jvf9OwemJd7VL4Jh4JepKCjyZR0PN0oFCHMxcvhhg1gsHh+T1hjajMFvUhBRXao+J8LrcFosMGBZl6oW0bbTEEvUlCRLlL/cyHO6Iu8Q4HwdhK1mYJepKAiWxH9z4UWips2gEHYJ3K2lYJepKDZVsQCM+DQyhyDMmb0gY2pzRT0IgUVLd3s98Kb0Q83bACDcLt920xBL1JQkT3nkAVjaA1GSYG+AAV9eBT0IgVtetMRL8SbjxRZjO1sGee2O0GNp+0U9CIFFTnpEcJsMCrSAAZhLjC3mYJepKBZF+lOZ6OfD63UUbQBDPLegIB+cbWdgl6koGE65tx2h+4GXaQQ3tkwRRvA/M+GMh5R0IsUVmThEhZKN4EE46wUVWRMAe4kajMFvUhBSTopPPuFgGb0h8Vn9Hu9Lkk6KeuSpCAFvUhByeiolKAPpcEoKbi4DP7o5aOyLkkKUtCLFJTddGSzhVgI72yYQcHtotnPdoL5xSUKepHCstLN9sY/H9qum2HBBjCAfm+boUo3wags6M3sRWb2fjN72MwumtndVT2XSJ2S9KhQKHa2jOt2wpkBl1W6OZxMSccK+xBUOaN/I/AG59yLgF/IPxZpnGE6KVS6gfzmI4HcqKPI3aU831OgWX0Yqgx6B3xN/vfrgc9X+FwitcmO9N28dAPZwWahNBgVPaQNoL+b/XuE8i6l7TZ/JZf7KeBdZvYrZL9Q/vFJ32Rm9wH3Adxxxx0VXo5I+dLxhMPJlH4JM/pQavTJKGsA62zZxo/h/z0G2nkThEJBb2YPArec8KX7ge8E/pVz7q1m9krgTcA9x7/ROXcBuABwcHDgilyPyLXmSxNFZr/+50PaR1+kWQqYvcNR6SYMhV5N59xVwe2Z2W8Dr8s//J/AbxZ5LpEQzRYud4uVbvq7XT771afLuKTCih5oBvOuWu2lD0OVNfrPA/8k//tLgb+q8LlEajGvZxcr3fQDWowtckSx5/891B0bhipr9D8M/Gcz6wIj8jq8SJPMg77gjD6g2wkm6eb3i/X8v0coY2q7yoLeOfc+4MVVPb5ICOY3HSlhe2Ugs98knfCsG3YKPYb/9whl3aHt1BkrUsCghC5S//OhNBgVbQAD2NsJ60TOtlPQixQw7yItXrpZfLw6ZX0BxYJ+a8uCKke1nYJepIAySzfZ49U/o886fYtXdfd6um9sKBT0IgXMTnrcKb6PPnu8ercj+gawoqUb0M1HQqKgFylgmI7Z2+mwVaCLFBZvPlLvjN4//6b3v12koA+Hgl6kgGRUvIsUwmkwKqsBLHsMBX0oFPQiBSQlNBfBwtkwNS9e+tJR0QYwyMpZWowNg4JepIAkLX5cAIRzNsz87B7N6JtEQS9SQJI2rHTjZ/QljGlfNfpgKOhFCsgWY4uH4nXbYZwNk8xm9CWUbvITOZ3TobR1U9CLFDAoaTE2lAajshrAIHtXMJ460vG08GNJMQp6kQLKOADM2+t1gindFG0Ag4XeAC3I1k5BL7Ih5xzDEoO+H8DBZslsH305DVOgg81CoKAX2VA6njKeulJKN5DtXa/7EDB/zk3RBjBYOL9HQV87Bb3Ihsq4ifaifgBnw2Q3HSletgEFfUgU9CIbmi9clle6qX0xtsxS1G44J3K2nYJeZEPJ7OTKshZj6993Pih1cVkz+lAo6EU25AOsjM5Y/zh1h+KwpAYwmP+71D0mUdCLbGx+AFh5pY6k5gajMm464s27fRX0dVPQi2xoeFh+6WZSc4NRWYe0AZzb7rBl2l4ZAgW9yIZ8I1CZpZvFx61DWYe0AZgZe72uGqYCoKAX2dCwgsXYxce91nwDWFnjAd8EpqCvm4JeZENJOsYMrivhbkxQ/77zshvAQHeZCoWCXmRDg3zh0qx4FynMFy/rKnWUXYoCnUkfCgW9yIbKPOcG6j8bpuxSFGhGHwoFvciGyuwihfpLN2Uf6eAfS52x9VPQi2yozK2I0Myg39NibBAKBb2ZvcLMHjWzqZkdHPvaz5nZY2b2STP77mKXKRKeJB2zX+bCZc0NRmU3gEH2S6PuEzml+Iz+EeCfA+9d/KSZPR94FfD1wMuB/25m5WxNEAlEMirnNoKebzCqq9RR9tk9MN9eqdsJ1qvQK+qc+wRw0q6De4EHnHMp8Ndm9hhwN/DnRZ7vNH/2qUv8uz/6eBUPLXKqx78y5IW33VDa4/kGo9/5wGd416NfLO1xV/X3l/Mbg5e862bq4GW/+l7K2ZvUXP/yJbfzQ9/63Eoeu7xX9ErPAt6/8PHn8s9dxczuA+4DuOOOOzZ6sn6vy1039zf6WZFNPe/mfV5xcFupj/mTL72Lj3z2b0t9zHXcev05btrvlfZ4L3v+zXz8808xnuq+scvc2C/v3/24pUFvZg8Ct5zwpfudc+8oegHOuQvABYCDg4ON3t+9+NnP4MXPfnHRSxGp3Q9/WzUzurp87fk+/+XV31j3ZbTe0qB3zt2zweM+Ady+8PFt+edEROQaq2p75TuBV5lZz8yeA9wFfLCi5xIRkTMU3V75/Wb2OeCbgT82s3cBOOceBd4CfBz438CPOefqvb29iEhLFd1183bg7ad87ZeAXyry+CIiUpw6Y0VEGk5BLyLScAp6EZGGU9CLiDSchXQGhZldAj6z4Y/fCHy5xMsJQdPG1LTxQPPG1LTxQPPGdNJ4nu2cO3/aDwQV9EWY2UXn3MHy74xH08bUtPFA88bUtPFA88a0yXhUuhERaTgFvYhIwzUp6C/UfQEVaNqYmjYeaN6YmjYeaN6Y1h5PY2r0IiJysibN6EVE5AQKehGRhmtE0JvZy/ObkD9mZj9b9/WUwcweN7OPmdnDZnax7utZl5m92cyeNLNHFj73TDN7j5n9Vf7nM+q8xnWdMqbXm9kT+ev0sJn90zqvcR1mdruZ/amZfdzMHjWz1+Wfj/J1OmM8Mb9Gu2b2QTP7aD6mN+Sff46ZfSDPvP9hZjtnPk7sNfr8puOfAl5GdsvCDwGvds5FfRNZM3scOHDORdnoYWbfBiTAbzvnXpB/7o3AV51zv5z/Qn6Gc+5n6rzOdZwyptcDiXPuV+q8tk2Y2a3Arc65D5vZPvAQ8H3ADxDh63TGeF5JvK+RAXvOucTMtoH3Aa8D/jXwNufcA2b268BHnXO/dtrjNGFGfzfwmHPu0865Q+ABspuTS42cc+8Fvnrs0/cCv5X//bfI/iOMxiljipZz7gvOuQ/nfx8AnyC7t3OUr9MZ44mWyyT5h9v5/xzwUuAP8s8vfY2aEPTPAj678PGpNyKPjAPebWYP5TdQb4KbnXNfyP/+ReDmOi+mRD9uZn+Rl3aiKHMcZ2Z3At8IfIAGvE7HxgMRv0Zm1jGzh4EngfcA/w/4O+fcOP+WpZnXhKBvqm9xzv0j4HuAH8vLBo3hspph3HXDzK8BXwu8CPgC8B/qvZz1mVkfeCvwU865pxa/FuPrdMJ4on6NnHMT59yLyO69fTfwD9Z9jCYEfSNvRO6ceyL/80myu3jdXe8VleJLeR3V11OfrPl6CnPOfSn/D3EK/AaRvU553fetwO86596Wfzra1+mk8cT+GnnOub8D/pTs1q03mJm/Q+DSzGtC0H8IuCtfhd4BXkV2c/JomdlevpiEme0B3wU8cvZPReGdwGvzv78WeEeN11IKH4i57yei1ylf6HsT8Ann3H9c+FKUr9Np44n8NTpvZjfkfz9HtunkE2SB/y/yb1v6GkW/6wYg3y71n4AO8Ob8frXRMrPnMr8Xbxf4vdjGZGa/D3w72ZGqXwJ+EfhDspvG30F2HPUrnXPRLG6eMqZvJysJOOBx4EcW6ttBM7NvAf4v8DFgmn/658nq2tG9TmeM59XE+xq9kGyxtUM2MX+Lc+7f5hnxAPBM4CPAa5xz6amP04SgFxGR0zWhdCMiImdQ0IuINJyCXkSk4RT0IiINp6AXEWk4Bb2ISMMp6EVEGu7/A/R4AvxoS2VkAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAD4CAYAAAAZ1BptAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dfZBV9Z3n8feHbkB8BKWHpQAHE9mkSHaChlFSSc1mtKJopoJTZVJaU5HNsmF2xdqkJrsJzm6tebI2mdmEiTXGWicSMZWJGvMgZaEMgybZTEqkjQRFZej4ECAoLU8+AP1w73f/OL+G0+29fZuHPrcP9/OquvQ53/M75/xO96W//Xs49ygiMDMzq2VcsytgZmZjl5OEmZnV5SRhZmZ1OUmYmVldThJmZlZXe7MrcLJNnTo1Zs+e3exqmJmVypNPPvlaRHQMjZ9ySWL27Nl0dnY2uxpmZqUi6eVacXc3mZlZXSNOEpLaJD0l6aG0foGkDZK6JN0naUKKT0zrXWn77Nwxbk7xrZKuzMUXpliXpOW5eM1zmJlZMY6lJfEZ4Lnc+teBFRFxIbAPWJLiS4B9Kb4ilUPSXOA64D3AQuDbKfG0AbcDVwFzgetT2eHOYWZmBRhRkpA0E/go8J20LuAy4IFUZBVwTVpelNZJ2y9P5RcB90ZET0S8CHQBl6RXV0S8EBG9wL3AogbnMDOzAoy0JfF3wOeBalo/D9gfEf1pfQcwIy3PALYDpO0HUvkj8SH71IsPd45BJC2V1Cmps7u7e4SXZGZmjTRMEpL+DNgdEU8WUJ/jEhF3RsT8iJjf0fG2GVxmZnacRjIF9oPAxyRdDZwGnA18C5gsqT39pT8T2JnK7wRmATsktQPnAHty8QH5fWrF9wxzDjMzK0DDlkRE3BwRMyNiNtnA86MR8RfAY8C1qdhi4MG0vDqtk7Y/Gtnnka8Grkuzny4A5gBPABuBOWkm04R0jtVpn3rnMLMWdbivwgNP7sCPOSjGidwn8QXgryR1kY0f3JXidwHnpfhfAcsBImILcD/wLPAIsCwiKqmVcBOwlmz21P2p7HDnMLMW9TePbOW//fA3/Gyrxx+LcEx3XEfEz4CfpeUXyGYmDS1zGPh4nf1vBW6tEV8DrKkRr3kOM2tdu984DMAbPf0NStrJ4DuuzcysLicJMyuVgZEINbUWrcNJwszKJWUJOUsUwknCzMzqcpIws1IJPPW1SE4SZlZK8qhEIZwkzKxUfA9dsZwkzKxUwgPXhXKSMLNSco4ohpOEmZnV5SRhZqXi2U3FcpIws1LxmESxnCTMrKScJYrgJGFmpeLOpmI5SZhZKbm7qRhOEmZmVlfDJCHpNElPSPqNpC2SvpTid0t6UdKm9JqX4pJ0m6QuSZslXZw71mJJ29JrcS7+fklPp31uk7K/ESSdK2ldKr9O0pST/y0wszI5MnDd3Gq0jJG0JHqAyyLifcA8YKGkBWnbf4+Ieem1KcWuInt+9RxgKXAHZL/wgVuAS8meNndL7pf+HcCnc/stTPHlwPqImAOsT+tm1tKyLCH3NxWiYZKIzJtpdXx6DTd2tAi4J+33ODBZ0nTgSmBdROyNiH3AOrKEMx04OyIej+zJ5vcA1+SOtSotr8rFzcysACMak5DUJmkTsJvsF/2GtOnW1KW0QtLEFJsBbM/tviPFhovvqBEHmBYRu9LyK8C0OvVbKqlTUmd3tx+ObnYq8wf8FWtESSIiKhExD5gJXCLpvcDNwLuBPwbOBb4warXM6hDUacFExJ0RMT8i5nd0dIxmNcysyfz40mId0+ymiNgPPAYsjIhdqUupB/gu2TgDwE5gVm63mSk2XHxmjTjAq6k7ivR197HU18xOXR6SKMZIZjd1SJqclicBHwGez/3yFtlYwTNpl9XADWmW0wLgQOoyWgtcIWlKGrC+Alibtr0uaUE61g3Ag7ljDcyCWpyLm1mLCvc3Fap9BGWmA6sktZEllfsj4iFJj0rqIGv1bQL+cyq/Brga6AIOAp8CiIi9kr4CbEzlvhwRe9PyjcDdwCTg4fQC+Bpwv6QlwMvAJ473Qs3s1OKWRDEaJomI2AxcVCN+WZ3yASyrs20lsLJGvBN4b434HuDyRnU0M7PR4TuuzaxUjg5cuylRBCcJMyuV8PSmQjlJmJlZXU4SZlYqnttULCcJMysl9zYVw0nCzErF90kUy0nCzErJnwJbDCcJMysltyiK4SRhZqXkFFEMJwkzK5UjDQhniUI4SZhZqYSzQ6GcJMyslJwsiuEkYWalMtDd5HHrYjhJmFkpOUkUw0nCzErFyaFYThJmVioDYxHOFcUYyeNLT5P0hKTfSNoi6UspfoGkDZK6JN0naUKKT0zrXWn77Nyxbk7xrZKuzMUXpliXpOW5eM1zmJn5ZrpijKQl0QNcFhHvA+YBC9Ozq78OrIiIC4F9wJJUfgmwL8VXpHJImgtcB7wHWAh8W1Jbeizq7cBVwFzg+lSWYc5hZi2qwltM6HiY/qg0uyotoWGSiMybaXV8egVwGfBAiq8CrknLi9I6afvlyj5kZRFwb0T0RMSLZM/AviS9uiLihYjoBe4FFqV96p3DzFrUrrYfMnHqz3l63y+bXZWWMKIxifQX/yZgN7AO+C2wPyL6U5EdwIy0PAPYDpC2HwDOy8eH7FMvft4w5xhav6WSOiV1dnd3j+SSzKykquoFIKLa5Jq0hhEliYioRMQ8YCbZX/7vHtVaHaOIuDMi5kfE/I6OjmZXx8zslHFMs5siYj/wGPABYLKk9rRpJrAzLe8EZgGk7ecAe/LxIfvUi+8Z5hxm1qrSgLXHrYsxktlNHZImp+VJwEeA58iSxbWp2GLgwbS8Oq2Ttj8a2TSE1cB1afbTBcAc4AlgIzAnzWSaQDa4vTrtU+8cZtaqlLKDnydRiPbGRZgOrEqzkMYB90fEQ5KeBe6V9FXgKeCuVP4u4HuSuoC9ZL/0iYgtku4HngX6gWUR2fQESTcBa4E2YGVEbEnH+kKdc5hZqwsniSI0TBIRsRm4qEb8BbLxiaHxw8DH6xzrVuDWGvE1wJqRnsPMzCmiGL7j2sxKZeAmOg9JFMNJwsxKxumhSE4SZlZS7nAqgpOEmZXKkaeXeg5sIZwkzKxksuQgT4EthJOEmZXMQAvCSaIIThJmVk6+T6IQThJmVipHxiQ8y6kQThJmVjLubiqSk4SZlZJTRDGcJMyslDwDthhOEmZWKuHupkI5SZhZKbkhUQwnCTMrGaeHIjlJmFnJpDuu3d1UCCcJMysVj0kUaySPL50l6TFJz0raIukzKf5FSTslbUqvq3P73CypS9JWSVfm4gtTrEvS8lz8AkkbUvy+9BhT0qNO70vxDZJmn8yLNzOz4Y2kJdEPfC4i5gILgGWS5qZtKyJiXnqtAUjbrgPeAywEvi2pLT3+9HbgKmAucH3uOF9Px7oQ2AcsSfElwL4UX5HKmVlLSw8d8tBEIRomiYjYFRG/TstvAM8BM4bZZRFwb0T0RMSLQBfZI0gvAboi4oWI6AXuBRYp+yjHy4AH0v6rgGtyx1qVlh8ALpc/+tGstaXk4DGJYhzTmETq7rkI2JBCN0naLGmlpCkpNgPYntttR4rVi58H7I+I/iHxQcdK2w+k8mbW4tyQKMaIk4SkM4EfAZ+NiNeBO4B3AvOAXcA3RqWGI6vbUkmdkjq7u7ubVQ0zK0Ac6W5ymijCiJKEpPFkCeL7EfFjgIh4NSIqEVEF/oGsOwlgJzArt/vMFKsX3wNMltQ+JD7oWGn7Oan8IBFxZ0TMj4j5HR0dI7kkMys5fwpsMUYyu0nAXcBzEfHNXHx6rtifA8+k5dXAdWlm0gXAHOAJYCMwJ81kmkA2uL06sj8HHgOuTfsvBh7MHWtxWr4WeDT854NZi8t+BVT9q6AQ7Y2L8EHgk8DTkjal2F+TzU6aR/YTewn4S4CI2CLpfuBZsplRyyKiAiDpJmAt0AasjIgt6XhfAO6V9FXgKbKkRPr6PUldwF6yxGJmhkclitEwSUTEL6l918qaYfa5Fbi1RnxNrf0i4gWOdlfl44eBjzeqo5m1jiNjEk2uR6vwHddmVkrZcKiNNicJMysltySK4SRhZiXjKbBFcpIws5KJIV9tNDlJmFkpeQpsMZwkzKxUjrYjnCSK4CRhZiXjMYkiOUmYmVldThJmVkrubiqGk4SZlYy7m4rkJGFmpXL0YzmcJIrgJGFmpeSWRDGcJMyspJwkiuAkYWYlk32wn1NEMZwkzKyU3N1UDCcJMyslD1wXw0nCzEpmYApsk6vRIkbyjOtZkh6T9KykLZI+k+LnSlonaVv6OiXFJek2SV2SNku6OHesxan8NkmLc/H3S3o67XNbeq523XOYWevyFNhijaQl0Q98LiLmAguAZZLmAsuB9RExB1if1gGuAuak11LgDsh+4QO3AJeSPar0ltwv/TuAT+f2W5ji9c5hZi3OSaIYDZNEROyKiF+n5TeA54AZwCJgVSq2CrgmLS8C7onM48BkSdOBK4F1EbE3IvYB64CFadvZEfF4ZCNR9ww5Vq1zmFmL88B1MY5pTELSbOAiYAMwLSJ2pU2vANPS8gxge263HSk2XHxHjTjDnGNovZZK6pTU2d3dfSyXZGal44cOFWnESULSmcCPgM9GxOv5bakFMKo/seHOERF3RsT8iJjf0dExmtUwszHC3U3FGFGSkDSeLEF8PyJ+nMKvpq4i0tfdKb4TmJXbfWaKDRefWSM+3DnMrEUdaUe4u6kQI5ndJOAu4LmI+GZu02pgYIbSYuDBXPyGNMtpAXAgdRmtBa6QNCUNWF8BrE3bXpe0IJ3rhiHHqnUOM2tZkfvXRlv7CMp8EPgk8LSkTSn218DXgPslLQFeBj6Rtq0Brga6gIPApwAiYq+krwAbU7kvR8TetHwjcDcwCXg4vRjmHGbWsjwFtkgNk0RE/BJQnc2X1ygfwLI6x1oJrKwR7wTeWyO+p9Y5zMzc3VQM33FtZmZ1OUmYWckMdDdVm1yP1uAkYWal5N6mYjhJmFkpeeC6GE4SZlYqR5KDmxKFcJIws5LxFNgiOUmYWSk5SRTDScLMSsm9TcVwkjCzknF3U5GcJMysVI5+ULiTRBGcJMzMrC4nCTMrmYE7rd2SKIKThJmVkgeui+EkYWal5M9uKoaThJmVTJrd5JZEIZwkzKxUIje/yUbfSB5fulLSbknP5GJflLRT0qb0ujq37WZJXZK2SroyF1+YYl2SlufiF0jakOL3SZqQ4hPTelfaPvtkXbSZlZ+nwBZjJC2Ju4GFNeIrImJeeq0BkDQXuA54T9rn25LaJLUBtwNXAXOB61NZgK+nY10I7AOWpPgSYF+Kr0jlzMwAP5muKA2TRET8AtjbqFyyCLg3Inoi4kWy51xfkl5dEfFCRPQC9wKLJAm4DHgg7b8KuCZ3rFVp+QHg8lTezFqa77gu0omMSdwkaXPqjpqSYjOA7bkyO1KsXvw8YH9E9A+JDzpW2n4glX8bSUsldUrq7O7uPoFLMjOzvONNEncA7wTmAbuAb5y0Gh2HiLgzIuZHxPyOjo5mVsXMCuKWRDGOK0lExKsRUYmIKvAPZN1JADuBWbmiM1OsXnwPMFlS+5D4oGOl7eek8mbW0gamwDpJFOG4koSk6bnVPwcGZj6tBq5LM5MuAOYATwAbgTlpJtMEssHt1ZH9lB8Drk37LwYezB1rcVq+Fng0/K4wM49JFKq9UQFJPwA+DEyVtAO4BfiwpHlkP62XgL8EiIgtku4HngX6gWURUUnHuQlYC7QBKyNiSzrFF4B7JX0VeAq4K8XvAr4nqYts4Py6E75aMyu9ADSwYKOuYZKIiOtrhO+qERsofytwa434GmBNjfgLHO2uyscPAx9vVD8za01uSRTDd1ybWcn4jusiOUmYWWnkhyXdkiiGk4SZlUYcGZBwkiiKk4SZlUbk/3WOKISThJmVRnVQZnCWKIKThJmVRpYjfJ9EkZwkzKw0ssSQkoP7mwrhJGFmpREBUn5kwkabk4SZlYa7m4rnJGFmpTGou8kK4SRhZqVRzbUknCyK4SRhZqURETAwJuGB60I4SZhZafguieI5SZhZaUQV3N1ULCcJMyuNINfd5CRRCCcJMyuNQVNgPSZRiIZJQtJKSbslPZOLnStpnaRt6euUFJek2yR1Sdos6eLcPotT+W2SFufi75f0dNrnNkka7hxm1rqyz25ycijSSFoSdwMLh8SWA+sjYg6wPq0DXEX2XOs5wFLgDsh+4ZM99vRSsqfQ3ZL7pX8H8OncfgsbnMPMWlSQv+PayaIIDZNERPyC7BnTeYuAVWl5FXBNLn5PZB4HJkuaDlwJrIuIvRGxD1gHLEzbzo6IxyNrO94z5Fi1zmFmLapazSUGdzcV4njHJKZFxK60/AowLS3PALbnyu1IseHiO2rEhzvH20haKqlTUmd3d/dxXI6ZlUE1m94EuCVRlBMeuE4tgFH9aTU6R0TcGRHzI2J+R0fHaFbFzJoonySsGMebJF5NXUWkr7tTfCcwK1duZooNF59ZIz7cOcysRVX8jOvCHW+SWA0MzFBaDDyYi9+QZjktAA6kLqO1wBWSpqQB6yuAtWnb65IWpFlNNww5Vq1zmFmLqlTd3VS09kYFJP0A+DAwVdIOsllKXwPul7QEeBn4RCq+Brga6AIOAp8CiIi9kr4CbEzlvhwRA4PhN5LNoJoEPJxeDHMOO8m63+hh14FD/NHMyc2uitmwquS6mzxwXYiGSSIirq+z6fIaZQNYVuc4K4GVNeKdwHtrxPfUOoedfB9Z8XP2H+zjpa99tNlVMRtWdVBLworgO66N/Qf7ml0FsxGpDBq4dpoogpOEmZVGVD1wXTQnCTMrjQoeuC6ak4SZlUZ+dpMVw0nCjhj0kQdmY1DV90kUzknCaDvjeSZ0PEK/k8SIfPKuDaxY96/NrkZLGnTHtafAFsJJwjj9/LuZOPVnVJwkRuT/bXuNb63f1uxqtKSqZzcVzknCjuh3f6+NcRXfJ1E4J4kWlx+HcEvCxjqPQxTPSaLFHeqrHFn2mISNdfmb6fz40mI4SbS4fJLoq1SGKWnWfNWqxySK5iTR4vJdTD2V/ibWxKyxQQ+ma141WoqTRIvLdzH19vkznGxsq0S+tes0UQQniRbX33+0+d5TcZKwsS0/DlH1mEQhnCRa3Ft9B48s9zpJ2BhXdZIoXMPnSdip7WD/0SThlkRj1Wqg8XuJ6sRmV6Ul5bub/LzrYpxQS0LSS5KelrRJUmeKnStpnaRt6euUFJek2yR1Sdos6eLccRan8tskLc7F35+O35X21YnU194uP1jd54HrhioRnHnh33DmO/+22VVpSW5JFO9kdDf9aUTMi4j5aX05sD4i5gDr0zrAVcCc9FoK3AFZUiF7JOqlwCXALQOJJZX5dG6/hSehvpbT23+09eDZTY0NzAZT2+Em16Q15VsPFSeJQozGmMQiYFVaXgVck4vfE5nHgcmSpgNXAusiYm9E7APWAQvTtrMj4vH0WNR7cseyk6R3UEvC3U2N+K705hr0KbD+WRTiRJNEAP8k6UlJS1NsWkTsSsuvANPS8gxge27fHSk2XHxHjbidRPnB6p5+J4lGfFd6c1VzYxIVj0kU4kQHrj8UETsl/QGwTtLz+Y0REZJG/X9VSlBLAc4///zRPt0ppb96tCXR6+6mhtySaC6PSRTvhFoSEbEzfd0N/IRsTOHV1FVE+ro7Fd8JzMrtPjPFhovPrBGvVY87I2J+RMzv6Og4kUtqOb25j+LorTpJNOIk0VxOEsU77iQh6QxJZw0sA1cAzwCrgYEZSouBB9PyauCGNMtpAXAgdUutBa6QNCUNWF8BrE3bXpe0IM1quiF3LDtJ8uMQHpNozEmiuar+gL/CnUh30zTgJ2lWajvwjxHxiKSNwP2SlgAvA59I5dcAVwNdwEHgUwARsVfSV4CNqdyXI2JvWr4RuBuYBDycXnYS5VsPngLbmO8laa58knBLohjHnSQi4gXgfTXie4DLa8QDWFbnWCuBlTXincB7j7eO1tiglkTVvwAb6et3Im2myqAk4YHrIvhjOVpcnweuj8nhSm+zq9DahoxJVN39N+qcJFpcfy4x9HvguqFetySaamgXU58fuTvqnCRaXL4l0eck0VCvu+SaatC9EQp6+50kRpuTRIvrq+YfX+ok0Uj+hkPPrine21oSFf8MRpuTRItzS+LY5O9Q93TY4gVHWw5SxS2JAjhJtLiKk8QxyXc3+SM6ilfJj0Gol76Kk8Roc5JocYNaEh6UbcgtieYa6OJrYyIa10ePWxKjzkmixeXHIQ73e3pnI/nZTfmPNLFiVFN30/hxp8G4Xnc3FcBJosXlk4SfJ9FY/oZDf2pu8Qbui5g4bhJSL/sO+g+b0eYk0eIGtyR6mliTcsjfcOjvV/EGWhKnjz8dxvXx+/2HmlyjU5+TRIvLPzP4UO5511ZbX+6Oa3fPFW9gCuyZE85A43rZdcBPCBxtThJNsHnHfh59/tVmVwMYPHB9uOIk0Uj+++XupuINfF7T6e2no3F9/H6/37Oj7UQfOmTH4WN//y8AvPS1jza5JlBJN9OpOoke/B+ukfyYxP5D/iu2aJGSxGltpwPw5O92ExGkT6O2UeCWRBNM6HiYSed/p9nVAKA/sr+M2zmD3or7dxvJj0n8bt/eYUraaBjobprUPgmArtf2se7ZsdEqP1U5STTBxKk/p/2MLg72Nn820cDNdO06g75wS6KR/lxL4rf7dgxT0kbD0SSRtSTOP6+dv127lX7fVDdqnCSa6JUDbza7ClSiQsQ4xmsS/eHuk0byYxIvv769iTVpTQMfyzGQJP7Dh2awbfebPPzMK82s1inNSaKJPv/TXza7ClSiH2IcE8ZNooK7mxp5s+fo9+j3b/6+iTVpTQMfy3HeaVMBOH/aW/zheaez8l9ebGa1TmljPklIWihpq6QuScubXZ8TdSjXxfRs5e85cLC5M2Qq1QqijYnjTqcityQaeenAdiLGMYl/Q3fP75pdnZbz0p63APiTWZdw1viz+MHz/8j1l07lqd/t51ddrzW5dqemMZ0kJLUBtwNXAXOB6yXNbW6tjt2h3n5+0fUyv9i2ix9vfv5IvG3STlZu/FUTa5a6T2IcZ004k9Bb/GLbLj/tq4aIoK9SZdeh3zGh2sHsM/4dve1dPPnya+zcf4hdBw7x6uuHOXCwj57+ij9G/CSJCN443Me2V9/gwU072fjybgDOOu00bpx3Ixte2cDd2z9Nx6xHWf7TX7HnzZ4j++0/2Mv2vQfZ/cZh9h/s5c2efg73Vfz+PkYay29mSR8AvhgRV6b1mwEi4n/X22f+/PnR2dl5zOe6+TsfY3P8dlBs6HcmNGSjBhcKUmyIQxJ72wdveFcfbB0Pp1WDKRUYF4N3HTh83rFO8hvJT3bHBPEH/cH/7BnPfz2jn9MrwZlVGB/Hfr6Toehzvu1nzNt+rIPWutvFH/fAov5xfP6sYEI1OKsKiqN/cTW6hhPfXruWI6EGu5xo3Y5F1PmaP8nQba+Mz77Lj+wPZvT2sLV9HHeeMYF1EwIBU/uzn0Nbk96/zfYf33EjH7/8xuPaV9KTETF/aHys3ycxA8iPDu4ALh1aSNJSYCnA+eeff1wnmj5lNrvTlMaIIMgep3v0jRbpP1gWURY6uqwjxd725hwfMLlvPHuqfUxA/Nv20/kvk2fxXN+bfPfN3/MWFYbOzYg6vwAG6hSKXNaq/z9fNdbyU8r/sFdcP7GDf3/2FP5Pz17+uX8fB6NCX91fkydmJMdpWCaXQXM/jdzmY6vtwDGUX1d+y9EzvDPE9WdP44MTJvPm4d38uvIGvapSVTakWo0ggvT+GVoTjahuI639sf5Mhis/KOXU+e0aw28+pvoc/V7X+gkOXm+TGDdOTGgbxwfa2vjI+KnMOPtcaJ/AuyL4RqWXF/rf5KcHX+UFDtEfVaqCcYJxUvp5ZLUb+Js40srJ/hN56N/cx3/7Rv5AIz/I1LOnHu8J6xrrLYlrgYUR8Z/S+ieBSyPipnr7HG9LwsysldVrSYzpMQlgJzArtz4zxczMrABjPUlsBOZIukDSBOA6YHWT62Rm1jLG9JhERPRLuglYC7QBKyNiS5OrZWbWMsZ0kgCIiDXAmmbXw8ysFY317iYzM2siJwkzM6vLScLMzOpykjAzs7rG9M10x0NSN/Dyce4+FThVPiXM1zI2nSrXcqpcB/haBvxhRHQMDZ5ySeJESOqsdcdhGflaxqZT5VpOlesAX0sj7m4yM7O6nCTMzKwuJ4nB7mx2BU4iX8vYdKpcy6lyHeBrGZbHJMzMrC63JMzMrC4nCTMzq8tJIpG0UNJWSV2Slje7Po1IWilpt6RncrFzJa2TtC19nZLiknRburbNki5uXs0HkzRL0mOSnpW0RdJnUryM13KapCck/SZdy5dS/AJJG1Kd70sfe4+kiWm9K22f3cz6DyWpTdJTkh5K66W8DgBJL0l6WtImSZ0pVsb32GRJD0h6XtJzkj4w2tfhJEH2nwG4HbgKmAtcL2luc2vV0N3AwiGx5cD6iJgDrE/rkF3XnPRaCtxRUB1Hoh/4XETMBRYAy9L3vozX0gNcFhHvA+YBCyUtAL4OrIiIC4F9wJJUfgmwL8VXpHJjyWeA53LrZb2OAX8aEfNy9xGU8T32LeCRiHg38D6yn8/oXkdEtPwL+ACwNrd+M3Bzs+s1gnrPBp7JrW8Fpqfl6cDWtPx/getrlRtrL+BB4CNlvxbgdODXZM9kfw1oH/peI3tOygfScnsqp2bXPdVnZvqFcxnwENmDlkt3HbnreQmYOiRWqvcYcA7w4tDv7Whfh1sSmRnA9tz6jhQrm2kRsSstvwJMS8uluL7UTXERsIGSXkvqotkE7AbWAb8F9kdEfyqSr++Ra0nbDwDnFVvjuv4O+DxQTevnUc7rGBDAP0l6UtLSFCvbe+wCoBv4buoG/I6kMxjl63CSOEVF9qdDaeY3SzoT+BHw2Yh4Pb+tTNcSEZWImEf2l/glwLubXKVjJunPgN0R8WSz63ISfSgiLibrglkm6U/yG0vyHmsHLgbuiIiLgLc42rUEjM51OElkdgKzcuszU6xsXpU0Ha+mBiMAAAGaSURBVCB93Z3iY/r6JI0nSxDfj4gfp3Apr2VAROwHHiPrlpksaeApkPn6HrmWtP0cYE/BVa3lg8DHJL0E3EvW5fQtyncdR0TEzvR1N/ATsgRetvfYDmBHRGxI6w+QJY1RvQ4nicxGYE6avTEBuA5Y3eQ6HY/VwOK0vJisf38gfkOa7bAAOJBrnjaVJAF3Ac9FxDdzm8p4LR2SJqflSWRjK8+RJYtrU7Gh1zJwjdcCj6a/BJsqIm6OiJkRMZvs/8KjEfEXlOw6Bkg6Q9JZA8vAFcAzlOw9FhGvANslvSuFLgeeZbSvo9mDMWPlBVwN/CtZH/L/aHZ9RlDfHwC7gD6yvzCWkPUDrwe2Af8MnJvKimz21m+Bp4H5za5/7jo+RNY83gxsSq+rS3otfwQ8la7lGeB/pfg7gCeALuCHwMQUPy2td6Xt72j2NdS4pg8DD5X5OlK9f5NeWwb+f5f0PTYP6EzvsZ8CU0b7OvyxHGZmVpe7m8zMrC4nCTMzq8tJwszM6nKSMDOzupwkzMysLicJMzOry0nCzMzq+v+FIhVK+U4d1wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(rews)\n",
    "figure()\n",
    "plot([x[0] for x in m.losses])\n",
    "plot([x[1] for x in m.losses])\n",
    "plot([x[-3] for x in m.losses])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.0513851 , 0.02443472, 0.07519621],\n",
       "       [0.16723773, 0.10752013, 0.14870928],\n",
       "       [0.13597797, 0.1101034 , 0.17943546]], dtype=float32)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show starting policy\n",
    "\n",
    "obs = [0,0,0,0,0,0,0,0,0,  0,1]\n",
    "policy, value = m.ft(m.ht(obs))\n",
    "np.reshape(policy, (3,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "turn 1\n",
      "[[ 1 -1  1]\n",
      " [-1  1 -1]\n",
      " [ 0  0  0]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([[0.0365698 , 0.03462234, 0.07436159],\n",
       "        [0.18296465, 0.091771  , 0.11462648],\n",
       "        [0.14332601, 0.13091904, 0.19083901]], dtype=float32),\n",
       " array([[-1.9384444, -1.7125351, -1.7163184],\n",
       "        [-1.885684 , -1.9812845, -1.9258984],\n",
       "        [-2.026409 , -1.7050903, -1.9425305]], dtype=float32))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test a \"challenging\" scenario\n",
    "\n",
    "obs = [1, -1, 1, -1, 1, -1, 0, 0, 0,  0,1]\n",
    "TicTacToe(obs).render()\n",
    "policy, value = m.ft(m.ht(obs))\n",
    "\n",
    "# expect 6 and 8 to light up in reward...they don't\n",
    "rew = [m.gt(m.ht(obs), i)[0] for i in range(9)]\n",
    "np.reshape(policy, (3,3)), np.reshape(rew, (3,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}