ml/primary_model_pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Primary Model Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time\n",
    "import os\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import *\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.models import load_model\n",
    "from tensorflow.keras.utils import plot_model\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy import signal\n",
    "pd.set_option('display.max_columns', 500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.0.0'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Datasets and Clean\n",
    "In this configuration the relevant data set should be loaded from the same folder as the notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('/nfs/2018/j/jcruz-y-/neurotron_datasets/joined/joined_data_106979_24-Oct-19_17:31_jose_all_1.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The data consists of timestamps from the two hardware devices and a diff between them. When the two hardware data streams were stitched together an effor was made to minimize this diff, but the driver configuration did not easily permit eliminating it. This information is included to understand the accuracy of the data, but will not be used during the training.\n",
    "\n",
    "The time data is followed by the 8 channels from the Myo, this data will be used as input features.\n",
    "\n",
    "This is followed by the 63 positional points from the Leap cameras. These will be used as labels."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop(labels=[\"Leap timestamp\", \"timestamp diff\", \"emg timestamp\"], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "df = df[:5000]\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FFT\n",
    "**Transforming to spectrograms**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y axis corresponds to frequencies (units are hz?)\n",
    "# x axis corresponds to times (units are samples?)\n",
    "# Each square corresponds to the magnitude (the absolute value the wave differs from 0)\n",
    "shape = (6, 21, 8)\n",
    "flat_dim = shape[0]*shape[1]*shape[2]\n",
    "nperseg = 10\n",
    "noverlap = 8\n",
    "reshape = (-1, shape[0], shape[1], shape[2])\n",
    "j = 0\n",
    "i = 0\n",
    "dir_num = 0\n",
    "df_train_g = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Spectrogram hyperparameters\n",
    "nperseg=40 #increasing nperseg increases frequency resolution and decreases time res\n",
    "noverlap=11 #increasing time resolution requires increasing overlap\n",
    "window = 'hann' #hann provides good time and frequency resolution properties\n",
    "\n",
    "#Spectrogram\n",
    "frequencies, times, spec = signal.spectrogram(x=df['ch1'][24:48],\n",
    "                                              fs=50, nperseg=nperseg,\n",
    "                                              noverlap=noverlap, window=window)\n",
    "print('frequencies', frequencies.shape)\n",
    "print('times', times.shape)\n",
    "#print(spec)\n",
    "#Subplot 1\n",
    "fig = plt.figure(figsize=(14, 8))\n",
    "ax1 = fig.add_subplot(211)\n",
    "ax1 = plt.pcolormesh(times, frequencies, spec)\n",
    "ax1 = plt.show\n",
    "\n",
    "#Subplot 2\n",
    "frequencies, times, spec = signal.spectrogram(x=df['ch2'][24:48], fs=50, nperseg=nperseg, noverlap=noverlap, window=window)\n",
    "log_spec = np.log(spec)\n",
    "fig = plt.figure(figsize=(14, 8))\n",
    "ax1 = fig.add_subplot(211)\n",
    "ax1 = plt.pcolormesh(times, frequencies, spec)\n",
    "ax1 = plt.show\n",
    "\n",
    "#Subplot 3\n",
    "frequencies, times, spec = signal.spectrogram(x=df['ch3'][24:48], fs=50, nperseg=nperseg, noverlap=noverlap, window=window)\n",
    "log_spec = np.log(spec)\n",
    "fig = plt.figure(figsize=(14, 8))\n",
    "ax1 = fig.add_subplot(211)\n",
    "ax1 = plt.pcolormesh(times, frequencies, spec)\n",
    "ax1 = plt.show\n",
    "\n",
    "#Subplot 4\n",
    "frequencies, times, spec = signal.spectrogram(x=df['ch4'][24:48], fs=50, nperseg=nperseg, noverlap=noverlap, window=window)\n",
    "log_spec = np.log(spec)\n",
    "fig = plt.figure(figsize=(14, 8))\n",
    "ax1 = fig.add_subplot(211)\n",
    "ax1 = plt.pcolormesh(times, frequencies, spec)\n",
    "ax1 = plt.show\n",
    "\n",
    "\n",
    "#Subplot 5\n",
    "frequencies, times, spec = signal.spectrogram(x=df['ch5'][24:48], fs=50, nperseg=nperseg, noverlap=noverlap, window=window)\n",
    "log_spec = np.log(spec)\n",
    "fig = plt.figure(figsize=(14, 8))\n",
    "ax1 = fig.add_subplot(211)\n",
    "ax1 = plt.pcolormesh(times, frequencies, spec)\n",
    "ax1 = plt.show\n",
    "\n",
    "#Subplot 6\n",
    "frequencies, times, spec = signal.spectrogram(x=df['ch6'][24:48], fs=50, nperseg=nperseg, noverlap=noverlap, window=window)\n",
    "log_spec = np.log(spec)\n",
    "fig = plt.figure(figsize=(14, 8))\n",
    "ax1 = fig.add_subplot(211)\n",
    "ax1 = plt.pcolormesh(times, frequencies, spec)\n",
    "ax1 = plt.show\n",
    "\n",
    "#Subplot 7\n",
    "frequencies, times, spec = signal.spectrogram(x=df['ch7'][24:48], fs=50, nperseg=nperseg, noverlap=noverlap, window=window)\n",
    "log_spec = np.log(spec)\n",
    "fig = plt.figure(figsize=(14, 8))\n",
    "ax1 = fig.add_subplot(211)\n",
    "ax1 = plt.pcolormesh(times, frequencies, spec)\n",
    "ax1 = plt.show\n",
    "\n",
    "#Subplot 8\n",
    "frequencies, times, spec = signal.spectrogram(x=df['ch8'][24:48], fs=50, nperseg=nperseg, noverlap=noverlap, window=window)\n",
    "log_spec = np.log(spec)\n",
    "fig = plt.figure(figsize=(14, 8))\n",
    "ax1 = fig.add_subplot(211)\n",
    "ax1 = plt.pcolormesh(times, frequencies, spec)\n",
    "ax1 = plt.show\n",
    "\n",
    "#Specs and Shapes\n",
    "#print('label:', label_dir)\n",
    "#print('file:', file)\n",
    "print('spec_shape:', spec.shape)\n",
    "print('frequencies_shape:', frequencies.shape)\n",
    "print('times_shape:', times.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "channels = ['ch1', 'ch2', 'ch3', 'ch4', 'ch5', 'ch6', 'ch7', 'ch8']\n",
    "shape = (7, 13, 8)\n",
    "flat_dim = shape[0]*shape[1]*shape[2]\n",
    "nperseg = 12\n",
    "noverlap = 11\n",
    "reshape = (-1, shape[0], shape[1], shape[2])\n",
    "j = 1\n",
    "i = 0\n",
    "dir_num = 0\n",
    "df_train_g = pd.DataFrame()\n",
    "\n",
    "img = np.zeros(shape)\n",
    "k = 0\n",
    "num_samples = len(df)\n",
    "x_shape = (num_samples, 7, 13, 8)\n",
    "x1_train = np.zeros(x_shape)\n",
    "\n",
    "for i in range(len(df) - 24):\n",
    "    for j in range(8):\n",
    "        frequencies, times, spec = signal.spectrogram(x=df[channels[j]][k:k+24], fs=50,\n",
    "                                                  nperseg=nperseg, noverlap=noverlap,\n",
    "                                                  window='hann')\n",
    "        log_spec = np.log(spec)\n",
    "        img[:,:,j] = spec\n",
    "        j = j + 1\n",
    "    x1_train[i,:,:,:] = img\n",
    "    i = i + 1\n",
    "    k = k + 1\n",
    "    if i == 4457:\n",
    "        print(df.iloc[i,:])\n",
    "    print(i)\n",
    "print(x1_train.shape)\n",
    "flat = np.reshape(x1_train, (num_samples, flat_dim))\n",
    "df_train1 = pd.DataFrame(data=flat)\n",
    "df_train_g = df_train_g.append(df_train1, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_g.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_features(x_train):\n",
    "    shape = (7, 13, 8)\n",
    "    reshape = (-1, shape[0], shape[1], shape[2])\n",
    "    x_train = x_train.replace(-np.inf, 0)\n",
    "    x_train = x_train.replace(np.inf, 0)\n",
    "    #x_train = np.log(x_train.values)\n",
    "    x_train = x_train.values\n",
    "    x_train_norm = x_train.reshape(reshape)\n",
    "    return x_train_norm\n",
    "\n",
    "features = preprocess_features(df_train_g)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "x1_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'features' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-ce5d0f26b013>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mfeatures\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'features' is not defined"
     ]
    }
   ],
   "source": [
    "features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = df.loc[:, 'Wrist x':].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Conv_LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(106956, 24, 7, 13, 8)\n"
     ]
    }
   ],
   "source": [
    "seq_length = 24\n",
    "\n",
    "def reshape_overlap_samples(seq_length, feats, labels):\n",
    "    samples = feats.shape[0] - (seq_length - 1)    # all samples except the last sequence\n",
    "    shape = (24, 7, 13, 8)\n",
    "    new_f = np.zeros(shape=(samples, 24, 7, 13, 8))\n",
    "    print(new_f.shape)\n",
    "    new_l = labels[seq_length - 1:]\n",
    "    queue = collections.deque(feats[:seq_length])\n",
    "    new_f[0] = np.array(queue)\n",
    "    j = 1\n",
    "    for i in range(seq_length, feats.shape[0]):\n",
    "        queue.popleft()\n",
    "        queue.append(feats[i])\n",
    "        new_f[j] = np.array(queue)\n",
    "        j += 1\n",
    "    return (new_f, new_l)\n",
    "\n",
    "features_ol, labels_ol = reshape_overlap_samples(seq_length, features, labels)\n",
    "print(features_ol.shape)\n",
    "print(labels_ol.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_11 (InputLayer)        (None, 24, 7, 13, 8)      0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_v1_28 (B (None, 24, 7, 13, 8)      32        \n",
      "_________________________________________________________________\n",
      "conv_lst_m2d_24 (ConvLSTM2D) (None, 24, 7, 13, 16)     6208      \n",
      "_________________________________________________________________\n",
      "dropout_24 (Dropout)         (None, 24, 7, 13, 16)     0         \n",
      "_________________________________________________________________\n",
      "conv_lst_m2d_25 (ConvLSTM2D) (None, 7, 13, 32)         24704     \n",
      "_________________________________________________________________\n",
      "dropout_25 (Dropout)         (None, 7, 13, 32)         0         \n",
      "_________________________________________________________________\n",
      "flatten_10 (Flatten)         (None, 2912)              0         \n",
      "_________________________________________________________________\n",
      "dense_29 (Dense)             (None, 63)                183519    \n",
      "=================================================================\n",
      "Total params: 214,463\n",
      "Trainable params: 214,447\n",
      "Non-trainable params: 16\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "def create_model(input_shape):\n",
    "    inputlayer = Input(shape=input_shape)\n",
    "    \n",
    "    norm_input = BatchNormalization()(inputlayer)\n",
    "    model = ConvLSTM2D(16, kernel_size=2, padding='same', activation='relu', return_sequences=True)(norm_input)\n",
    "    model = Dropout(rate=0.3)(model)\n",
    "    #model = ConvLSTM2D(16, kernel_size=2, padding='same', activation='relu', return_sequences=True)(model)\n",
    "    #model = Dropout(rate=0.5)(model)\n",
    "    #model = ConvLSTM2D(32, kernel_size=2, padding='same', activation='relu', return_sequences=True)(model)\n",
    "    #model = Dropout(rate=0.5)(model)\n",
    "    model = ConvLSTM2D(32, kernel_size=2, padding='same', activation='relu', return_sequences=False)(model)\n",
    "    model = Dropout(rate=0.3)(model)\n",
    "    #model = ConvLSTM2D(128, kernel_size=2, padding='same', activation='relu', return_sequences=True)(model)\n",
    "    #model = Dropout(rate=0.5)(model)\n",
    "    #model = ConvLSTM2D(128, kernel_size=2, padding='same', activation='relu', return_sequences=False)(model)\n",
    "    #model = Dropout(rate=0.5)(model)\n",
    "    model = Flatten()(model)\n",
    "    #mode1 = BatchNormalization()(Dense(128, activation='relu')(model))\n",
    "    mode1 = BatchNormalization()(Dense(128, activation='relu')(model))\n",
    "    model = Dense(63, activation='None')(model)\n",
    "    #dense_5 = Dense(63, activation='relu')(model)\n",
    "    #model = TimeDistributed(MaxPooling2D(pool_size=(2, 2)))(model)\n",
    "    #model = Dropout(rate=0.2)(model)\n",
    "    #model = ConvLSTM2D(64, kernel_size=3, padding='same', activation='relu')(model)\n",
    "    #model = ConvLSTM2D(64, kernel_size=3, padding='same', activation='relu')(model)\n",
    "    #model = TimeDistributed(MaxPooling2D(pool_size=(2, 2)))(model)\n",
    "    #model = Dropout(rate=0.2)(model)\n",
    "    #model = Conv2D(128, kernel_size=3, padding='same', activation='relu')(model)\n",
    "    #model = Conv2D(128, kernel_size=3, padding='same', activation='relu')(model)\n",
    "    #model = TimeDistributed(MaxPooling2D(pool_size=(1, 1)))(model)\n",
    "    #model = Flatten()(model) \n",
    "\n",
    "    #dense_1 = BatchNormalization()(Dense(128, activation='relu')(model))\n",
    "    #dense_2 = BatchNormalization()(Dense(128, activation='relu')(dense_1))\n",
    "    #dense_1 = Dense(nclass, activation='softmax')(dense_1)\n",
    "\n",
    "    #lstm layers\n",
    "    #inputs = Input(shape=(None, 128), name=\"inputs\")(dense_2)\n",
    "    #lstm_0 = LSTM(100, return_sequences=True, name=\"lstm_0\")(dense_2)\n",
    "    #do = Dropout(0.4)(lstm_0)\n",
    "    #lstm_1 = LSTM(100, return_sequences=True, name=\"lstm_1\")(do)\n",
    "    #do_2 = Dropout(0.4)(lstm_1)\n",
    "    #lstm_1 = LSTM(100, return_sequences=False, name=\"lstm_1\")(do_2)\n",
    "    \n",
    "    #dense_3 = Dense(128, activation='relu', name=\"dense_3\")(lstm_1)\n",
    "    #dense_4 = BatchNormalization()(Dense(128, activation='relu')(dense_3))\n",
    "    #dense_5 = Dense(128, activation='relu')(dense_4)\n",
    "    #dense_6 = BatchNormalization()(Dense(128, activation='relu')(dense_5))\n",
    "    #dense_5 = Dense(64, activation='relu')(dense_4)\n",
    "    \n",
    "    #model_fc.add(Dense(100, input_dim=64))\n",
    "    #model_fc.add(Activation('relu'))\n",
    "    #model_fc.add(BatchNormalization())\n",
    "    #model_fc.add(Dropout(0.2))\n",
    "    #model_fc.add(Dense(64, input_dim=64))\n",
    "    #model_fc.add(Activation('relu'))\n",
    "    #model_fc.add(Dropout(0.2))\n",
    "    #model_fc.add(Dense(63, input_dim=64))\n",
    "    \n",
    "    #decoder layers\n",
    "    #decoder_0 = decoder.get_layer(\"decoder_0\")(lstm_out)\n",
    "    #decoder_0.trainable = False\n",
    "    #decoder_1 = decoder.get_layer(\"decoder_1\")(decoder_0)\n",
    "    #decoder_1.trainable = False\n",
    "    #decoder_output = decoder.get_layer(\"decoder_output\")(decoder_1)\n",
    "    #decoder_output.trainable = False\n",
    "    \n",
    "    model = Model(inputs=inputlayer, outputs=model)\n",
    "    #model = Model(inputs, decoder_output, name=\"model_v1\")\n",
    "    return model\n",
    "\n",
    "shape = (24, 7, 13, 8)\n",
    "model = create_model(shape)\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.keras.optimizers.Adam(lr=0.001)\n",
    "model.compile(optimizer=optimizer, loss='mse')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  928/19977 [>.............................] - ETA: 2:36 - loss: 5118735.4261"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-70-a1a5187099d0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeatures_ol\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels_ol\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mevaluate\u001b[0;34m(self, x, y, batch_size, verbose, sample_weight, steps, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1009\u001b[0m           \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1010\u001b[0m           \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1011\u001b[0;31m           steps=steps)\n\u001b[0m\u001b[1;32m   1012\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1013\u001b[0m   def predict(self,\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mmodel_iteration\u001b[0;34m(model, inputs, targets, sample_weights, batch_size, epochs, verbose, callbacks, val_inputs, val_targets, val_sample_weights, shuffle, initial_epoch, steps_per_epoch, validation_steps, mode, validation_in_fit, **kwargs)\u001b[0m\n\u001b[1;32m    327\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    328\u001b[0m         \u001b[0;31m# Get outputs.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 329\u001b[0;31m         \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    330\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    331\u001b[0m           \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/tensorflow/python/keras/backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   3074\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3075\u001b[0m     fetched = self._callable_fn(*array_vals,\n\u001b[0;32m-> 3076\u001b[0;31m                                 run_metadata=self.run_metadata)\n\u001b[0m\u001b[1;32m   3077\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_fetch_callbacks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3078\u001b[0m     return nest.pack_sequence_as(self._outputs_structure,\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1437\u001b[0m           ret = tf_session.TF_SessionRunCallable(\n\u001b[1;32m   1438\u001b[0m               \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1439\u001b[0;31m               run_metadata_ptr)\n\u001b[0m\u001b[1;32m   1440\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1441\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.evaluate(features_ol, labels_ol, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 15981 samples, validate on 3996 samples\n",
      "Epoch 1/2\n",
      "  408/15981 [..............................] - ETA: 7:39 - loss: 1657.2771"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-71-5ba552f8a91b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m history = model.fit(features_ol, labels_ol, batch_size=seq_length, epochs=2\n\u001b[0;32m----> 2\u001b[0;31m                     , verbose=1, validation_split=0.2, shuffle=False)\n\u001b[0m",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m    878\u001b[0m           \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    879\u001b[0m           \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 880\u001b[0;31m           validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m    881\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    882\u001b[0m   def evaluate(self,\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mmodel_iteration\u001b[0;34m(model, inputs, targets, sample_weights, batch_size, epochs, verbose, callbacks, val_inputs, val_targets, val_sample_weights, shuffle, initial_epoch, steps_per_epoch, validation_steps, mode, validation_in_fit, **kwargs)\u001b[0m\n\u001b[1;32m    327\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    328\u001b[0m         \u001b[0;31m# Get outputs.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 329\u001b[0;31m         \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    330\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    331\u001b[0m           \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/tensorflow/python/keras/backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   3074\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3075\u001b[0m     fetched = self._callable_fn(*array_vals,\n\u001b[0;32m-> 3076\u001b[0;31m                                 run_metadata=self.run_metadata)\n\u001b[0m\u001b[1;32m   3077\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_fetch_callbacks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3078\u001b[0m     return nest.pack_sequence_as(self._outputs_structure,\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1437\u001b[0m           ret = tf_session.TF_SessionRunCallable(\n\u001b[1;32m   1438\u001b[0m               \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1439\u001b[0;31m               run_metadata_ptr)\n\u001b[0m\u001b[1;32m   1440\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1441\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "history = model.fit(features_ol, labels_ol, batch_size=seq_length, epochs=2\n",
    "                    , verbose=1, validation_split=0.2, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXxc5X3v8c9Xo9W2ZBtbNosAG8ckMRBcohKa5YaEhDhkIXllAVIKIRBnI0lvs5H23kC2JvSWkMWU1LRmSW6gNEtDGwjN1nKbQLBMDNhQEgMGZGzLC1jeJFvS7/5xzthjeaQzWmZkWd/363VemnnOc848xwZ9/SxzjiICMzOzwVSNdQPMzOzQ57AwM7NMDgszM8vksDAzs0wOCzMzy+SwMDOzTA4Ls1EiaY6kkFRdQt33SvqvkZ7HrFIcFjYhSVoraY+kmf3Kf5f+op4zNi0zOzQ5LGwiexK4IP9G0inApLFrjtmhy2FhE9l3gIsK3l8M3FJYQdJUSbdI2iTpKUn/S1JVui8n6W8lbZb0BPCmIsf+o6T1ktZJ+pKk3FAbKeloSXdI2ippjaT3F+w7XVKbpE5JGyV9LS2vl/RdSVskPS9puaTZQ/1sszyHhU1k9wFNkl6c/hI/H/huvzrfAqYCJwCvJgmXS9J97wfeDPwR0Aq8s9+xNwE9wAvSOmcDlw2jnbcB7cDR6Wf8taTXpvu+AXwjIpqAecDtafnFabuPBWYAHwR2D+OzzQCHhVm+d/F64FFgXX5HQYB8NiK2R8Ra4Brgz9Iq7wa+HhHPRMRW4CsFx84GzgH+PCJ2RkQHcG16vpJJOhZ4BfCZiOiKiJXAP7C/R7QXeIGkmRGxIyLuKyifAbwgInojYkVEdA7ls80KOSxsovsO8B7gvfQbggJmAjXAUwVlTwHHpK+PBp7pty/v+PTY9ekw0PPA3wOzhti+o4GtEbF9gDZcCpwI/Hc61PTmguu6G7hN0rOS/kZSzRA/22wfh4VNaBHxFMlE9znAD/vt3kzyL/TjC8qOY3/vYz3JME/hvrxngG5gZkRMS7emiDhpiE18FjhCUmOxNkTEHyLiApIQuhr4vqTJEbE3Ij4fEQuAl5MMl12E2TA5LMySf52/NiJ2FhZGRC/JHMCXJTVKOh74C/bPa9wOfExSi6TpwBUFx64H/h24RlKTpCpJ8yS9eigNi4hngN8AX0knrV+Stve7AJIulNQcEX3A8+lhfZJeI+mUdCitkyT0+oby2WaFHBY24UXE4xHRNsDujwI7gSeA/wK+ByxL991AMtTzIPAAB/dMLgJqgUeA54DvA0cNo4kXAHNIehk/Aq6MiJ+n+xYBqyXtIJnsPj8idgNHpp/XSTIX858kQ1NmwyI//MjMzLK4Z2FmZpkcFmZmlslhYWZmmRwWZmaWqWy3QJa0jGRtd0dEnFxQ/lHgI0Av8JOI+LSk1wNfJVk5sgf4VET8Mq3/UpLbJjQAdwIfjxJm5WfOnBlz5swZ1WsyMzucrVixYnNENBfbV8775d8ELKHgW7GSXgOcC5waEd2S8t9m3Qy8JSKelXQyyXLE/DdUrye5B89vScJiEXBX1ofPmTOHtraBVkOamVl/kp4aaF/ZhqEi4h5ga7/iDwFfjYjutE5H+vN3EfFsWmc10CCpTtJRQFNE3Jf2Jm4B3lauNpuZWXGVnrM4EXiVpN9K+k9Jf1ykzjuAB9JAOYbkbpt57ezvcZiZWYVU+rGN1cARwBnAHwO3SzohPwch6SSS+9ucPZyTS1oMLAY47rjjMmqbmVmpKh0W7cAP03C4X1IfyZ09N0lqIbmVwUUR8Xhafx3QUnB8CwW3kO4vIpYCSwFaW1sPmgTfu3cv7e3tdHV1jcrFHOrq6+tpaWmhpsY3GzWzkal0WPwL8BrgV5JOJFn9tFnSNOAnwBUR8et85YhYnz4B7AySCe6LSB5GMyzt7e00NjYyZ84cJI3oQg51EcGWLVtob29n7ty5Y90cMxvnyjZnIelW4F7ghZLaJV1KcgO2EyStInn618VpL+NykqeJfU7SynTLr5T6MMnDXtYAj1PCSqiBdHV1MWPGjMM+KAAkMWPGjAnTizKz8ipbzyK9x34xFxap+yXgSwOcpw04udi+4ZgIQZE3ka7VzMqr0sNQh7yNnV1USdTkRE2uipqcqK6qoqrKv3jNbOJyWBSICDZt76avyBfEc1X58Kiiump/kNTkqqjO/6zSgP+a37JlC2eddRYAGzZsIJfL0dycfFHy/vvvp7a2NrN9l1xyCVdccQUvfOELR3CVZmZD57AoIImTjm6ity/o6Qv29vaxtzf52ZN/3ddH196gp7eP/pEioDofHlVpryQNmNrJTdy3fAXVVeKLX/g8jY2NfPKTnzzg+IggIqiqKj6VdOONN5bnws3MMjgs+pFEdU5U56C+JjdgvYj9gdLTuz9Yenr72NsX7OntY+eePnr7Du6lbNrezW5qeLxjB+1PPcH7/+w8TnnJqax++EHu+MlPufqvv8SDK3/H7t27Oe+88/jc5z4HwCtf+UqWLFnCySefzMyZM/ngBz/IXXfdxaRJk/jxj3/MrFmzDvosM7PRMGHD4vP/uppHnu0c1XMuOLqJK99y0gFlfX1BT9/+Hsre3qChNkdNrgoE3T19rPn9Y3z+mr/jylP/iC7gvR//S6ZOn0709XLpu97Cy157DgsWLGBPbx/P7drDtt172LZtGy9/5av46698hU9+4hMsW7aMK664onjDzMxGaMKGRaVUVYnaqhy1BX/SjfU1TJlUy7zmKcS2KcybN493v/HV+3om199+M9+55SZ69vawYcOz/P6x/+b4eSfS05vMqTy1ZRf19Q3MWfgKVq3bxpEnLOB3y+9l7ead++dP0qGwvb19bN7RzRGTaj1Jb2bDNmHDon8PYCxNnjyZXFUVuSr4w9onuOHb13H//fczbdo0LrzwQqbXwYuOamJSbY65Myczf9YU6upqOWZ6Az29weT6Gvp6e9nb28euPUlPJm9jZzdv/dLPqcmJWY31zGqq48imemY3Hfh6dlMds5vqmVJX7SW3ZnaQCRsWh6rOzk4aGxtpampi/fr13H333SxatGjf/ppcFQ1pN2XG5Lrk55Q6GuurmT+7EYC+iH3zKD1barnqLQvYuL2bjZ1dbOzs4g8dO/ivP2xme3fPQZ8/qTbHkWmQzG6qT18nYZIPlubGukHnc8zs8OOwOMScdtppLFiwgBe96EUcf/zxvOIVrxjyOaokaqtFbXUVDbU53vuK4rf72NndQ8f2bjZs66JjexIkG7Z1s3F7Fx2dXTzw9HNs7OxmT0/fQcdOn1ST9kj290oK3x/ZVM+MKXXkPPRldlhQCQ+dG5daW1uj/8OPHn30UV784hePUYvGxkivOSJ4ftdeNm7vYmNnNxu3JaGycXsSLPmQSb6fcuCxVYLmxrqDeiez0h5LPlimNtR46MvsECBpRUS0FtvnnoUNShLTJ9cyfXItLzpy4Ho9vX1s2bkn7Z10sXF7Nx0Fr5/esovla7fy/K69Bx1bV11VpIdyYG/lyKZ6Gmo99GU2VhwWNiqqc1X7frG/pGXgel17e+no7E57Kl3pEFj3vtern+3kF492sHtv70HHNtZXDzo5n59PqclV+pleZoc/h4VVVH1NjuNmTOK4GZMGrBMRbO/uSXsm3fuGvZIhsCRo7nt8Bx3bu+npN/YlJRP/hUNes4usAJvupcRmQ+KwsEOOJJrqa2iqr+EFsxoHrNfXF/uGvjq27w+W5HUX67d18WD782zeseegY/NLiQeanM+HTGO9HxxlBg4LG8eqqkRzYx3NjXXA1AHr7enpY9OOtIeSTtBv6EzmVDZu7+L3G7cPuJR4cm3uoGGv/RP0dfv21VV7PsUObw4LO+zVVldxzLQGjpnWMGi9nd096XdRuvf1TvLDXhu3dbFiCEuJ9/dO9geLlxLbeOawqJDRuEU5wLJlyzjnnHM48shBlibZsEyuq+aE5imc0DxlwDqFS4k3bOtKJus7u9hQEDKPru9k846DlxLnqkTzlLrMVV9NDf4WvR16yhYWkpYBbwY6IuLkgvKPAh8BeoGfRMSn0/LPApem5R+LiLvT8kXAN4Ac8A8R8dVytbmcZsyYwcqVKwG46qqrmDJlykG3KC/FsmXLOO200xwWY+TApcRNA9bLLyXekP9eShom+WB5assu7s9YSnzwN+kLV4B5KbFVVjl7FjcBS4Bb8gWSXgOcC5waEd3552xLWgCcD5wEHA38XNKJ6WHXAa8H2oHlku6IiEfK2O6Ku/nmm7nuuuvYs2cPL3/5y1myZAl9fX1ccsklrFy5kohg8eLFzJ49m5UrV3LeeefR0NAwpB6JVVbhUuLBFC4lLhYsq9Zt4+ePbqRr78FDX0311QdNzh8wFDa1nplTvJTYRkc5n8F9j6Q5/Yo/BHw1IrrTOh1p+bnAbWn5k5LWAKen+9ZExBMAkm5L6448LO66AjY8POLTHODIU+CNQ+v4rFq1ih/96Ef85je/obq6msWLF3Pbbbcxb948Nm/ezMMPJ218/vnnmTZtGt/61rdYsmQJCxcuHN2225gYylLifUuH095JRxoqGzq7eOLxzYMuJT5yah2zGw+enM8Hi5cSW5ZKz1mcCLxK0peBLuCTEbEcOAa4r6Bee1oG8Ey/8pcNdHJJi4HFAMcdd9woNrt8fv7zn7N8+XJaW5Nv2O/evZtjjz2WN7zhDTz22GN87GMf401vehNnn332GLfUxkrhUuL8zSKLKVxKXNg7yW/rt3Wx8pnn2bKz9KXEB4TM1OSuxDYxVfpvvho4AjgD+GPgdkknjNbJI2IpsBSSe0MNWnmIPYByiQje97738cUvfvGgfQ899BB33XUX1113HT/4wQ9YunTpGLTQxovCpcQnH5O9lDiZoC+YnE9fl7KUuOjk/NS6fbfB91Liw0+lw6Id+GEkdy+8X1IfMBNYBxxbUK8lLWOQ8sPC6173Ot75znfy8Y9/nJkzZ7JlyxZ27txJQ0MD9fX1vOtd72L+/PlcdtllADQ2NrJ9+/YxbrWNZ8NZSpzvnSTDX8n7tqeeo6Ozmz29B8+nHDG5llmN+yfnZzfVHXgDyal1zJjspcTjSaXD4l+A1wC/Siewa4HNwB3A9yR9jWSCez5wPyBgvqS5JCFxPvCeCre5rE455RSuvPJKXve619HX10dNTQ3f/va3yeVyXHrppUQEkrj66qsBuOSSS7jssss8wW1lN5SlxBvSMOlI51AKQyZzKfHUembng2Vq/f6QmVrP7EYvJT5UlO0W5ZJuBc4k6TlsBK4EvgMsAxYCe0jmLH6Z1v8r4H1AD/DnEXFXWn4O8HWSpbPLIuLLpXy+b1GemIjXbIeent4+Nu/YU3RyvjBktu0+eClxfU26sqzfkx4LQ8ZLiUfHmNyiPCIuGGDXhQPU/zJwUBBExJ3AnaPYNDOrsOpcFUdOTXoLpw5SL7+UeENn1wGT8/lgKWUpcdI72b98eFZj2ktpqvNS4hHw0gYzO2SUupS4s6unSO9k/2T9mo5kKXFvkaXEM/Pfom/M907SyfqC10dMrvXQVz8TLizycwATweH6FESb2CQxtaGGqQ1DX0q8fwisi2cHWUpcm6tKnvKY9kgKeyf7QqZpYi0lnjhXCtTX17NlyxZmzJhx2AdGRLBlyxbq6wf/BrHZ4arUpcTdPb1s2t7d73sp+18/tmE79/x+MzsGWkqc9kiOnJrenqUgWA6npcQTKixaWlpob29n06ZNY92Uiqivr6elZZDH1pkZddU5WqZPomX6wENfADvyD+Tqt+or/3r52q2ZS4mPLDLslQ+ZQ30p8YQKi5qaGubOnTvWzTCzcWhKXTVTSlhK/NyuvQdNzhe+Xv1sspS4/yhxrkrMaky+jzJ73xBY/f6QaRrbpcQTKizMzMpJEkdMruWIybW8+KjB70q8eceeopPzGzu7WLtlJ799cuvgS4nzWxomhSFz/IzJo35tDgszsworXEo8mN17eunYfvDk/IY0VB5uf56fdXYdsJT4iMm1PPC/Xz/6bR71M5qZ2ahoqM1x/IzJg/YUCpcSb+jsYvee3rK0xWFhZjaOlbqUeKT8VUYzM8vksDAzs0wOCzMzy+SwMDOzTA4LMzPL5LAwM7NMDgszM8vksDAzs0xlCwtJyyR1SFpVUHaVpHWSVqbbOWl5jaSbJT0s6VFJny04ZpGkxyStkXRFudprZmYDK2fP4iZgUZHyayNiYbrlH5f6LqAuIk4BXgp8QNIcSTngOuCNwALgAkkLythmMzMromxhERH3AFtLrQ5MllQNNAB7gE7gdGBNRDwREXuA24Bzy9FeMzMb2FjMWVwu6aF0mGp6WvZ9YCewHnga+NuI2AocAzxTcGx7WlaUpMWS2iS1TZQHHJmZVUKlw+J6YB6wkCQYrknLTwd6gaOBucAnJJ0w1JNHxNKIaI2I1ubm5lFqspmZVTQsImJjRPRGRB9wA0lIALwH+GlE7I2IDuDXQCuwDji24BQtaZmZmVVQRcNC0lEFb98O5FdKPQ28Nq0zGTgD+G9gOTBf0lxJtcD5wB2Va7GZmUEZn2ch6VbgTGCmpHbgSuBMSQtJJrTXAh9Iq18H3ChpNSDgxoh4KD3P5cDdQA5YFhGry9VmMzMrTtH/qeGHidbW1mhraxvrZpiZjRuSVkREa7F9/ga3mZllcliYmVkmh4WZmWVyWJiZWSaHhZmZZXJYmJlZJoeFmZllcliYmVkmh4WZmWVyWJiZWSaHhZmZZXJYmJlZJoeFmZllcliYmVkmh4WZmWVyWJiZWSaHhZmZZSpbWEhaJqlD0qqCsqskrZO0Mt3OKdj3Ekn3Slot6WFJ9Wn5S9P3ayR9U5LK1WYzMyuunD2Lm4BFRcqvjYiF6XYngKRq4LvAByPiJJJnd+9N618PvB+Yn27FzmlmZmVUtrCIiHuArSVWPxt4KCIeTI/dEhG9ko4CmiLivkgeFn4L8LbytNjMzAYyFnMWl0t6KB2mmp6WnQiEpLslPSDp02n5MUB7wbHtaVlRkhZLapPUtmnTpvK03sxsAqp0WFwPzAMWAuuBa9LyauCVwJ+mP98u6ayhnjwilkZEa0S0Njc3j1KTzcysomERERsjojci+oAbgNPTXe3APRGxOSJ2AXcCpwHrgJaCU7SkZWZmVkEVDYt0DiLv7UB+pdTdwCmSJqWT3a8GHomI9UCnpDPSVVAXAT+uZJvNzCwZ/ikLSbeSrGqaKakduBI4U9JCIIC1wAcAIuI5SV8Dlqf77oyIn6Sn+jDJyqoG4K50MzOzClKyyOjw09raGm1tbWPdDDOzcUPSiohoLbbP3+A2M7NMDgszM8vksDAzs0wOCzMzy+SwMDOzTA4LMzPL5LAwM7NMDgszM8vksDAzs0wOCzMzy+SwMDOzTA4LMzPL5LAwM7NMDgszM8tUUlhImiepLn19pqSPSZpW3qaZmdmhotSexQ+AXkkvAJYCxwLfK1urzMzskFJqWPRFRA/Jo1C/FRGfAo4a7ABJyyR1SFpVUHaVpHWSVqbbOf2OOU7SDkmfLChbJOkxSWskXVH6pZmZ2WgpNSz2SroAuBj4t7SsJuOYm4BFRcqvjYiF6XZnv31fo+CxqZJywHXAG4EFwAWSFpTYZjMzGyWlhsUlwJ8AX46IJyXNBb4z2AERcQ+wtdSGSHob8CSwuqD4dGBNRDwREXuA24BzSz2nmZmNjpLCIiIeiYiPRcStkqYDjRFx9TA/83JJD6XDVNMBJE0BPgN8vl/dY4BnCt63p2VFSVosqU1S26ZNm4bZPDMz66/U1VD/IalJ0hHAA8ANkr42jM+7HpgHLATWA9ek5VeRDE/tGMY594mIpRHRGhGtzc3NIzmVmZkVqC6x3tSI6JR0GXBLRFwp6aGhflhEbMy/lnQD++c/Xga8U9LfANOAPkldwAqSlVd5LcC6oX6umZmNTKlhUS3pKODdwF8N98MkHRUR69O3bwdWAUTEqwrqXAXsiIglkqqB+ekcyTrgfOA9w/18MzMbnlLD4gvA3cCvI2K5pBOAPwx2gKRbgTOBmZLagSuBMyUtBAJYC3xgsHNERI+ky9PPzgHLImL1YMeYmdnoU0SMdRvKorW1Ndra2sa6GWZm44akFRHRWmxfqRPcLZJ+lH7JrkPSDyS1jG4zzczsUFXq9yxuBO4Ajk63f03LzMxsAig1LJoj4saI6Em3mwCvTTUzmyBKDYstki6UlEu3C4Et5WyYmZkdOkoNi/eRLJvdQPJluncC7y1Tm8zM7BBT6u0+noqIt0ZEc0TMioi3Ae8oc9vMzOwQMZIn5f3FqLXCzMwOaSMJC41aK8zM7JA2krA4PL/NZ2ZmBxn0dh+StlM8FAQ0lKVFZmZ2yBk0LCKisVINMTOzQ9dIhqHMzGyCcFiYmVkmh4WZmWVyWJiZWSaHhZmZZXJYmJlZprKFhaRl6YOSVhWUXSVpnaSV6XZOWv56SSskPZz+fG3BMS9Ny9dI+qYkf3PczKzCytmzuAlYVKT82ohYmG53pmWbgbdExCnAxcB3CupfD7wfmJ9uxc5pZmZlVLawiIh7gK0l1v1dRDybvl0NNEiqk3QU0BQR90XysPBbgLeVp8VmZjaQsZizuFzSQ+kw1fQi+98BPBAR3cAxQHvBvva0rChJiyW1SWrbtGnT6LbazGwCq3RYXA/MAxaSPETpmsKdkk4CrgY+MJyTR8TSiGiNiNbmZj/11cxstFQ0LCJiY0T0RkQfcANwen6fpBbgR8BFEfF4WrwOaCk4RUtaZmZmFVTRsEjnIPLeDqxKy6cBPwGuiIhf5ytExHqgU9IZ6Sqoi4AfV7DJZmZGxl1nR0LSrcCZwExJ7cCVwJmSFpLc9nwt+4ebLgdeAHxO0ufSsrMjogP4MMnKqgbgrnQzM7MKUrLI6PDT2toabW1tY90MM7NxQ9KKiGgtts/f4DYzs0wOCzMzy+SwMDOzTA4LMzPL5LAwM7NMDgszM8vksDAzs0wOCzMzy+SwMDOzTA4LMzPL5LAwM7NMDgszM8vksDAzs0wOCzMzy+SwMDOzTA4LMzPL5LAwM7NMZQsLScskdUhaVVB2laR1klam2zkF+z4raY2kxyS9oaB8UVq2RtIV5WqvmZkNrJw9i5uARUXKr42Ihel2J4CkBcD5wEnpMX8nKScpB1wHvBFYAFyQ1jUzswqqLteJI+IeSXNKrH4ucFtEdANPSloDnJ7uWxMRTwBIui2t+8goN9fMzAYxFnMWl0t6KB2mmp6WHQM8U1CnPS0bqLwoSYsltUlq27Rp02i328xswqp0WFwPzAMWAuuBa0bz5BGxNCJaI6K1ubl5NE9tZjahlW0YqpiI2Jh/LekG4N/St+uAYwuqtqRlDFJuZmYVUtGehaSjCt6+HcivlLoDOF9SnaS5wHzgfmA5MF/SXEm1JJPgd1SyzWZmVsaehaRbgTOBmZLagSuBMyUtBAJYC3wAICJWS7qdZOK6B/hIRPSm57kcuBvIAcsiYnW52mxmZsUpIsa6DWXR2toabW1tY90MM7NxQ9KKiGgtts/f4DYzs0wOCzMzy+SwMDOzTA4LMzPL5LAwM7NMDgszM8vksDAzs0wOCzMzy+SwMDOzTA4LMzPL5LAwM7NMDgszM8vksDAzs0wOCzMzy+SwMDOzTA4LMzPLVNawkLRMUoekVUX2fUJSSJqZvp8q6V8lPShptaRLCupeLOkP6XZxOdtsZmYHK3fP4iZgUf9CSccCZwNPFxR/BHgkIk4leRzrNZJqJR1B8kjWlwGnA1dKml7mdpuZWYGyhkVE3ANsLbLrWuDTJM/i3lcdaJQkYEp6XA/wBuBnEbE1Ip4DfkaRADIzs/Kp+JyFpHOBdRHxYL9dS4AXA88CDwMfj4g+4BjgmYJ67WlZsXMvltQmqW3Tpk2j33gzswmqomEhaRLwl8Dniux+A7ASOBpYCCyR1DSU80fE0ohojYjW5ubmEbfXzMwSle5ZzAPmAg9KWgu0AA9IOhK4BPhhJNYATwIvAtYBxxacoyUtMzOzCqloWETEwxExKyLmRMQckiGl0yJiA8lk91kAkmYDLwSeAO4GzpY0PZ3YPjstMzOzCin30tlbgXuBF0pql3TpINW/CLxc0sPAL4DPRMTmiNia7luebl9Iy8zMrEIUEdm1xqHW1tZoa2sb62aYmY0bklZERGuxff4Gt5mZZXJYmJlZJoeFmZllcliYmVkmh4WZmWVyWJiZWSaHhZmZZaoe6wYccm5+K0Qf1DQkW3UD1NTv/1kzCarr0331BfWKlRUcm6sBaayvzsxsWBwW/VXXQfd22NEBPV2wd/f+n3t3Q9/e4Z1XVUWCJh9CxQJn0oEhVbRskGNz/qs1s9Hj3yj9/ek/D76/r7cgQHbB3i7o2X3gz727igdN0bL0mK7n02N3H3i+vp7hXUdVdZFQ6R80xXpKg/WeBjm2Kje8dprZuOCwGKqqHNRNSbZK6N2bHTQH/ByorN+xu7ak+7sODL/oG147q2pKH5KrqS9eVnIvqwGqPN1mVkkOi0NdribZGNKjPYYnIgmnwYKm1J5SYS+rezvs2FS858Uw702WqysxaEoMn6z5KM832QTnsLD9JKiuTbb6qeX/vAjo3TNI0AwQPv1D6oAhwS7o2gY7NhYfJhyu6vqh95RKHc7rH2DVdQ4nO+Q4LGzsSMkvxuo6aJhW/s+LGCBoivWUis1HFavXBbu2Fj9fb/cwG6oSVtpl9ZQK62UEWK7W4WSZHBY2cUj7f4lWQl9fEh5Zw3QlDecVlO3cVHyOqnfP8NqpqhH2lIa4GCJXM7p/zlYRDguzcqmqgtpJyVYJfb2j0FMqUta1rd+xXSNcRp4r0isqpac0zMUQXkY+KvynaHa4qMpB7eRkq4TentHpKfU/dvdz+4f4Cs8XvcNrZ36lXilDcsPpKfWvd5iu1CtbWEhaBrwZ6IiIk/vt+wTwt0BzRGxOy84Evg7UAJsj4tVp+SLgG0AO+IeI+Gq52mxmQ5Crhlwj1DVW5vMOWEY+gp5S4bF7dsKuzcWXnA93GXmutsQl4sPsKRXWq66vWDiVs2dxE7AEuKWwUNKxwNnA0wVl04C/AxZFxNOSZqXlOavFV00AAAbXSURBVOA64PVAO7Bc0h0R8UgZ221mh6KKLyPfU8Jw3hDno7o6oaejeM9ruHJ1BwZI41HwvrtG788iVbawiIh7JM0psuta4NPAjwvK3gP8MCKeTo/tSMtPB9ZExBMAkm4DzgUcFmZWPoUr9SohAnq6iwRNKb2nfgFWpgUcFZ2zkHQusC4iHtSBS/VOBGok/QfQCHwjIm4BjgGeKajXDrxskPMvBhYDHHfccaPbeDOzcpHSYat6qNBivaGqWFhImgT8JckQVLF2vBQ4i+SP6l5J9w31MyJiKbAUoLW1dZhfDTYzs/4q2bOYB8wF8r2KFuABSaeT9Bi2RMROYKeke4BT0/JjC87RAqyrYJvNzIwKPvwoIh6OiFkRMSci5pAEwWkRsYFk/uKVkqrTHsjLgEeB5cB8SXMl1QLnA3dUqs1mZpYoW1hIuhW4F3ihpHZJlw5UNyIeBX4KPATcT7JEdlVE9ACXA3eThMftEbG6XG02M7PiFHF4Du23trZGW1vbWDfDzGzckLQiIlqL7Ts8v2poZmajymFhZmaZHBZmZpbpsJ2zkLQJeGqYh88ENo9ic8YDX/Phb6JdL/iah+r4iGgutuOwDYuRkNQ20CTP4crXfPibaNcLvubR5GEoMzPL5LAwM7NMDovilo51A8aAr/nwN9GuF3zNo8ZzFmZmlsk9CzMzy+SwMDOzTBM6LCQtkvSYpDWSriiyv07SP6X7fzvAk//GjRKu9y8kPSLpIUm/kHT8WLRzNGVdc0G9d0gKSeN+mWUp1yzp3enf9WpJ36t0G0dbCf9tHyfpV5J+l/73fc5YtHO0SFomqUPSqgH2S9I30z+PhySdNuIPjYgJuQE54HHgBKAWeBBY0K/Oh4Fvp6/PB/5prNtd5ut9DTApff2h8Xy9pV5zWq8RuAe4D2gd63ZX4O95PvA7YHr6ftZYt7sC17wU+FD6egGwdqzbPcJr/h/AacCqAfafA9wFCDgD+O1IP3Mi9yz2Pd87IvYA+ed7FzoXuDl9/X3gLPV7Huw4knm9EfGriNiVvr2P5GFT41kpf8cAXwSuBroq2bgyKeWa3w9cFxHPwQHPvB+vSrnmAJrS11OBZyvYvlEXEfcAWwepci5wSyTuA6ZJOmoknzmRw6LY872PGahOJM/W2AbMqEjrRl8p11voUpJ/mYxnmdecds+PjYifVLJhZVTK3/OJwImSfi3pPkmLKta68ijlmq8CLpTUDtwJfLQyTRszQ/3/PVMlH6tq44SkC4FW4NVj3ZZyklQFfA147xg3pdKqSYaiziTpPd4j6ZSIeH5MW1VeFwA3RcQ1kv4E+I6kkyOib6wbNl5M5J7FOrKf772vjqRqku7rloq0bvSVcr1Ieh3wV8BbI6K7Qm0rl6xrbgROBv5D0lqSsd07xvkkdyl/z+3AHRGxNyKeBH5PEh7jVSnXfClwO0BE3AvUk9xw73BV0v/vQzGRw6KU53vfAVycvn4n8MtIZ4/GoczrlfRHwN+TBMV4H8eGjGuOiG0RMTP2Pxf+PpJrH8+PWCzlv+t/IelVIGkmybDUE5Vs5Cgr5ZqfBs4CkPRikrDYVNFWVtYdwEXpqqgzgG0RsX4kJ5yww1AR0SMp/3zvHLAsIlZL+gLQFhF3AP9I0l1dQzKZdP7YtXhkSrze/wNMAf45ncd/OiLeOmaNHqESr/mwUuI13w2cLekRoBf4VESM1x5zqdf8CeAGSf+TZLL7veP4H35IupUk8Gem8zBXAjUAEfFtknmZc4A1wC7gkhF/5jj+8zIzswqZyMNQZmZWIoeFmZllcliYmVkmh4WZmWVyWJiZWSaHhdkwSeqVtLJgG/CutsM495yB7ihqNhYm7PcszEbB7ohYONaNMKsE9yzMRpmktZL+RtLDku6X9IK0fI6kXxY8L+S4tHy2pB9JejDdXp6eKifphvSZE/8uqWHMLsomPIeF2fA19BuGOq9g37aIOAVYAnw9LfsWcHNEvAT4v8A30/JvAv8ZEaeSPKNgdVo+n+RW4icBzwPvKPP1mA3I3+A2GyZJOyJiSpHytcBrI+IJSTXAhoiYIWkzcFRE7E3L10fETEmbgJbCGzcqeSrjzyJifvr+M0BNRHyp/FdmdjD3LMzKIwZ4PRSFd/3txXOMNoYcFmblcV7Bz3vT179h/80o/xT4f+nrX5A8xhZJOUlTK9VIs1L5Xypmw9cgaWXB+59GRH757HRJD5H0Di5Iyz4K3CjpUyS3x87fCfTjwFJJl5L0ID4EjOh20majzXMWZqMsnbNojYjNY90Ws9HiYSgzM8vknoWZmWVyz8LMzDI5LMzMLJPDwszMMjkszMwsk8PCzMwy/X8aVzl5voAjFAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot training & validation loss values\n",
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('Model loss')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Train', 'Test'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CNN + LSTM"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "    \n",
    "model = tf.keras.models.Sequential()\n",
    "model.add(TimeDistributed(Conv2D(32, kernel_size=2, padding='same', activation='relu', input_shape=(None, 7, 13, 8))))\n",
    "model.add(TimeDistributed(Flatten()))    \n",
    "model.add(LSTM(64, return_sequences=True, input_shape=(seq_length, 8)))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(LSTM(64))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dense(100, input_dim=64))\n",
    "model.add(Activation('relu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(64, input_dim=64))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(18, input_dim=64))\n",
    "\n",
    "\n",
    "model.compile(optimizer='Adam', loss='mse')\n",
    "#model.summary()\n",
    "history = model.fit(features, labels, batch_size=seq_length, epochs=5, verbose=1, validation_split=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7, 13, 8)\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_34 (InputLayer)        (None, 24, 7, 13, 8)      0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_v1_71 (B (None, 24, 7, 13, 8)      32        \n",
      "_________________________________________________________________\n",
      "time_distributed_53 (TimeDis (None, 24, 7, 13, 32)     1056      \n",
      "_________________________________________________________________\n",
      "time_distributed_54 (TimeDis (None, 24, 7, 13, 32)     4128      \n",
      "_________________________________________________________________\n",
      "batch_normalization_v1_72 (B (None, 24, 7, 13, 32)     128       \n",
      "_________________________________________________________________\n",
      "time_distributed_55 (TimeDis (None, 24, 2912)          0         \n",
      "_________________________________________________________________\n",
      "lstm_28 (LSTM)               (None, 24, 64)            762112    \n",
      "_________________________________________________________________\n",
      "lstm_29 (LSTM)               (None, 64)                33024     \n",
      "_________________________________________________________________\n",
      "dense_58 (Dense)             (None, 63)                4095      \n",
      "_________________________________________________________________\n",
      "batch_normalization_v1_73 (B (None, 63)                252       \n",
      "_________________________________________________________________\n",
      "dense_59 (Dense)             (None, 63)                4032      \n",
      "=================================================================\n",
      "Total params: 808,859\n",
      "Trainable params: 808,653\n",
      "Non-trainable params: 206\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "def create_model(input_shape):\n",
    "\n",
    "    inputlayer = Input(shape=input_shape)\n",
    "    norm_input = BatchNormalization()(inputlayer)\n",
    "    model = TimeDistributed(Conv2D(32, kernel_size=2, padding='same', activation='relu'))(norm_input)\n",
    "    model = TimeDistributed(Conv2D(32, kernel_size=2, padding='same', activation='relu'))(model)\n",
    "    model = BatchNormalization()(model)\n",
    "    #model = BatchNormalization()(TimeDistributed(Dense(128, activation='relu')(model)))\n",
    "    model = TimeDistributed(Flatten())(model)\n",
    "    model = LSTM(64, activation='relu', return_sequences=True)(model)\n",
    "    model = LSTM(64, activation='relu', return_sequences=False)(model)\n",
    "    model = BatchNormalization()(Dense(63, activation='relu')(model))\n",
    "    dense = Dense(63, activation='relu')(model)\n",
    "    \n",
    "    #norm_input = BatchNormalization()(inputlayer)\n",
    "    #model = Conv2D(32, kernel_size=2, padding='same', activation='relu')(norm_input)\n",
    "    #model = Conv2D(32, kernel_size=2, padding='same', activation='relu')(model)\n",
    "    #model = MaxPooling2D(pool_size=(2, 2))(model)\n",
    "    #model = Dropout(rate=0.2)(model)\n",
    "    #model = Conv2D(64, kernel_size=3, padding='same', activation='relu')(model)\n",
    "    #model = Conv2D(64, kernel_size=3, padding='same', activation='relu')(model)\n",
    "    #model = MaxPooling2D(pool_size=(2, 2))(model)\n",
    "    #model = Dropout(rate=0.2)(model)\n",
    "    #vmodel = Conv2D(128, kernel_size=3, padding='same', activation='relu')(model)\n",
    "    #model = Conv2D(128, kernel_size=3, padding='same', activation='relu')(model)\n",
    "    #model = MaxPooling2D(pool_size=(1, 1))(model)\n",
    "    #model = Flatten()(model) \n",
    "\n",
    "    #dense_1 = BatchNormalization()(Dense(128, activation='relu')(model))\n",
    "    #dense_2 = BatchNormalization()(Dense(128, activation='relu')(dense_1))\n",
    "    #dense_1 = Dense(nclass, activation='softmax')(dense_1)\n",
    "    \n",
    "    #lstm layers\n",
    "    #inputs = Input(shape=(None, 128))(dense_2)\n",
    "    #dense_3 = Dense(128, activation='relu', name=\"dense_3\")(dense_2)\n",
    "    #lstm_0 = LSTM(100, input_shape=(128,), return_sequences=True, name=\"lstm_0\")(dense_3)\n",
    "    #do = Dropout(0.4)(lstm_0)\n",
    "    #stm_1 = LSTM(100, return_sequences=True, name=\"lstm_1\")(do)\n",
    "    #do_2 = Dropout(0.4)(lstm_0)\n",
    "    #lstm_1 = LSTM(100, return_sequences=False, name=\"lstm_1\")(do_2)\n",
    "    \n",
    "    #dense_3 = Dense(128, activation='relu', name=\"dense_3\")(lstm_1)\n",
    "    #dense_4 = BatchNormalization()(Dense(128, activation='relu')(dense_3))\n",
    "    #dense_5 = Dense(128, activation='relu')(dense_4)\n",
    "    #dense_6 = BatchNormalization()(Dense(128, activation='relu')(dense_5))\n",
    "    #dense_5 = Dense(64, activation='relu')(dense_4)\n",
    "    \n",
    "    #model_fc.add(Dense(100, input_dim=64))\n",
    "    #model_fc.add(Activation('relu'))\n",
    "    #model_fc.add(BatchNormalization())\n",
    "    #model_fc.add(Dropout(0.2))\n",
    "    #model_fc.add(Dense(64, input_dim=64))\n",
    "    #model_fc.add(Activation('relu'))\n",
    "    #model_fc.add(Dropout(0.2))\n",
    "    #model_fc.add(Dense(63, input_dim=64))\n",
    "    \n",
    "    #decoder layers\n",
    "    #decoder_0 = decoder.get_layer(\"decoder_0\")(lstm_out)\n",
    "    #decoder_0.trainable = False\n",
    "    #decoder_1 = decoder.get_layer(\"decoder_1\")(decoder_0)\n",
    "    #decoder_1.trainable = False\n",
    "    #decoder_output = decoder.get_layer(\"decoder_output\")(decoder_1)\n",
    "    #decoder_output.trainable = False\n",
    "    \n",
    "    model = Model(inputs=inputlayer, outputs=dense)\n",
    "    #model = Model(inputs, decoder_output, name=\"model_v1\")\n",
    "    return model\n",
    "\n",
    "\n",
    "print(shape)\n",
    "model = create_model((24, 7, 13, 8))\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer='Adam', loss='mse')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "history = model.fit(features_ol, labels_ol, batch_size=seq_length, epochs=5, verbose=1, validation_split=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## conv_lstm_2d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#lstm layers\n",
    "inputs = Input(shape=(None, 8), name=\"inputs\")\n",
    "lstm_0 = ConvLSTM2D(100, return_sequences=True, name=\"lstm_0\")(inputs)\n",
    "do = Dropout(0.4)(lstm_0)\n",
    "lstm_1 = ConvLSTM2D(100, return_sequences=False, name=\"lstm_1\")(do)\n",
    "do_2 = Dropout(0.4)(lstm_0)\n",
    "lstm_1 = ConvLSTM2D(100, return_sequences=False, name=\"lstm_1\")(do_2)\n",
    "lstm_out = Dense(9, activation=None, name=\"lstm_out\")(lstm_1)\n",
    "\n",
    "#decoder layers\n",
    "decoder_0 = decoder.get_layer(\"decoder_0\")(lstm_out)\n",
    "decoder_0.trainable = False\n",
    "decoder_1 = decoder.get_layer(\"decoder_1\")(decoder_0)\n",
    "decoder_1.trainable = False\n",
    "decoder_output = decoder.get_layer(\"decoder_output\")(decoder_1)\n",
    "decoder_output.trainable = False\n",
    "\n",
    "model = Model(inputs, decoder_output, name=\"model_v1\")\n",
    "optimizer = tf.keras.optimizers.Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.00, amsgrad=False)\n",
    "model.compile(optimizer=optimizer, loss='mse')\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Autoencoder\n",
    "The final pipeline will pass the data into an LSTM and then into an autoencoder to expand features into the 63 point space. This autoencoder will be trained first below:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_ar = df.loc[:, 'ch1':'ch8'].values\n",
    "label_ar = df.loc[:, 'Wrist x':].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(106979, 8)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_ar.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(106979, 63)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_ar.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Build the autoencoder structure:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /nfs/2018/j/jcruz-y-/miniconda3/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /nfs/2018/j/jcruz-y-/miniconda3/lib/python3.7/site-packages/tensorflow/python/keras/utils/losses_utils.py:170: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 63)                0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 32)                2048      \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 16)                528       \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 9)                 153       \n",
      "_________________________________________________________________\n",
      "decoder_0 (Dense)            (None, 32)                320       \n",
      "_________________________________________________________________\n",
      "decoder_1 (Dense)            (None, 16)                528       \n",
      "_________________________________________________________________\n",
      "decoder_output (Dense)       (None, 63)                1071      \n",
      "=================================================================\n",
      "Total params: 4,648\n",
      "Trainable params: 4,648\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "encoding_dim = 9 #dimensionality of 'feature' vector\n",
    "\n",
    "a_fn = None\n",
    "\n",
    "# Full autoencoder\n",
    "input_vec = Input(shape=(63,))\n",
    "dense_0 = Dense(32, activation=a_fn)(input_vec)\n",
    "dense_1 = Dense(16, activation=a_fn)(dense_0)\n",
    "encoded = Dense(encoding_dim, activation=a_fn)(dense_1)\n",
    "dense_2 = Dense(32, activation=a_fn, name='decoder_0')(encoded)\n",
    "dense_3 = Dense(16, activation=a_fn, name='decoder_1')(dense_2)\n",
    "decoded = Dense(63, activation=a_fn, name='decoder_output')(dense_3)\n",
    "\n",
    "autoencoder = Model(input_vec, decoded)\n",
    "\n",
    "# Encoder from autoencoder\n",
    "encoder = Model(input_vec, encoded)\n",
    "\n",
    "# Decoder from autoencoder layers\n",
    "decoder_input = Input(shape=(encoding_dim,), name='encoded_input')\n",
    "decode_0 = autoencoder.layers[-3](decoder_input)\n",
    "decode_1 = autoencoder.layers[-2](decode_0)\n",
    "decode_output = autoencoder.layers[-1](decode_1)\n",
    "decoder = Model(decoder_input, decode_output, name='decoder')\n",
    "\n",
    "# Train Autoencoder\n",
    "optimizer = tf.keras.optimizers.Adam(lr=0.0005, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)\n",
    "autoencoder.compile(optimizer=optimizer, loss='mean_squared_error')\n",
    "autoencoder.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "106979/106979 [==============================] - 2s 22us/sample - loss: 2075.7868\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "2075.7868355574788"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "autoencoder.evaluate(label_ar, label_ar)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From the above evaluation the initial untrained loss is around 2500.\n",
    "Now train the model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 85583 samples, validate on 21396 samples\n",
      "Epoch 1/3\n",
      "85583/85583 [==============================] - 1s 10us/sample - loss: 27.4648 - val_loss: 30.2431\n",
      "Epoch 2/3\n",
      "85583/85583 [==============================] - 1s 10us/sample - loss: 25.0079 - val_loss: 27.6195\n",
      "Epoch 3/3\n",
      "85583/85583 [==============================] - 1s 10us/sample - loss: 23.3839 - val_loss: 26.2109\n"
     ]
    }
   ],
   "source": [
    "ret = autoencoder.fit(label_ar, label_ar, batch_size=312, epochs=3, verbose=1, validation_split=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now the encoder - decoder pair will be tested seperately for loss:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "106979/106979 [==============================] - 2s 17us/sample - loss: 23.4510\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "23.450966647503495"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded_vec = encoder.predict(label_ar)\n",
    "decoded_vec = decoder.predict(encoded_vec)\n",
    "\n",
    "#compilation is only to enable evaluation\n",
    "decoder.compile(optimizer='adam', loss='mse')\n",
    "decoder.evaluate(encoded_vec, label_ar)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(106979, 9)\n",
      "(106979, 63)\n"
     ]
    }
   ],
   "source": [
    "print(encoded_vec.shape)\n",
    "print(decoded_vec.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM\n",
    "Here the lstm will be trained. The 8 channels of sEMG data will be processed by the LSTM, which will return a vector of 9 complex features. These will then be processed by the autoencoder, trained in the previous section, to create the 63 hand coordinates.\n",
    "\n",
    "To train the sequential recurrent LSTM the data will be grouped into 'sequences' of 24 steps in time. The LSTM will be trained with many sequence groups."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'feature_ar' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-62d9b3fce720>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mnew_f\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_l\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0mfeatures\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moverlap_samples\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseq_length\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeature_ar\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel_ar\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'feature_ar' is not defined"
     ]
    }
   ],
   "source": [
    "seq_length = 24\n",
    "\n",
    "def overlap_samples(seq_length, feats, labels):\n",
    "    new_l = labels[seq_length - 1:]\n",
    "    feat_list = [feats[i:i + seq_length] for i in range(feats.shape[0] - seq_length + 1)]\n",
    "    new_f = np.array(feat_list)\n",
    "    return new_f, new_l\n",
    "\n",
    "features, labels = overlap_samples(seq_length, feature_ar, label_ar)\n",
    "print(features.shape)\n",
    "print(labels.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Attention\n",
    "**This model will incorporate a component of attention**\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model from phillip peremy\n",
    "https://github.com/philipperemy/keras-attention-mechanism/blob/master/attention_lstm.py\n",
    "Attention vector\n",
    "\n",
    "Also, sometimes the time series can be N-dimensional. It could be interesting to have one atention vector per dimension. Let's\n",
    "\n",
    "Attention can just then be a softmax applied to an output of something?\n",
    "\n",
    "The permute function switches the positions of the axis and the dims argument tells how you want the final positions to be.\n",
    "\n",
    "For example, if x is 4-dimensional and of the shape (None, 2, 4, 5, 8) - (None is the batch size here) and if you specify dims = (3, 2, 1, 4), then the following four steps will take place:\n",
    "\n",
    "1. Third dimension will move to first\n",
    "2. Second dimension will move to second\n",
    "3. First dimension will move to third\n",
    "4. Fourth dimension will move to fourth\n",
    "\n",
    "Remember, the indexing starts at 1 and not 0. The dimension zero is the batch size. So finally the output\n",
    "\n",
    "**RepeatVector**\n",
    "Repeats the input vector n times\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /nfs/2018/j/jcruz-y-/miniconda3/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 20, 2)        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lstm (LSTM)                     (None, 20, 32)       4480        input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "permute (Permute)               (None, 32, 20)       0           lstm[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "reshape (Reshape)               (None, 32, 20)       0           permute[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 32, 20)       420         reshape[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "attention_vec (Permute)         (None, 20, 32)       0           dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "add (Add)                       (None, 20, 32)       0           lstm[0][0]                       \n",
      "                                                                 attention_vec[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "flatten (Flatten)               (None, 640)          0           add[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 1)            641         flatten[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 5,541\n",
      "Trainable params: 5,541\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "INPUT_DIM = 2\n",
    "TIME_STEPS = 20\n",
    "# if True, the attention vector is shared across the input_dimensions where the attention is applied.\n",
    "SINGLE_ATTENTION_VECTOR = False\n",
    "APPLY_ATTENTION_BEFORE_LSTM = False\n",
    "\n",
    "def attention_3d_block(inputs):\n",
    "    # inputs.shape = (batch_size, time_steps, input_dim)\n",
    "    input_dim = int(inputs.shape[2])\n",
    "    a = Permute((2, 1))(inputs)\n",
    "    a = Reshape((input_dim, TIME_STEPS))(a) # this line is not useful. It's just to know which dimension is what.\n",
    "    a = Dense(TIME_STEPS, activation='softmax')(a)\n",
    "    if SINGLE_ATTENTION_VECTOR:\n",
    "        a = Lambda(lambda x: K.mean(x, axis=1), name='dim_reduction')(a)\n",
    "        a = RepeatVector(input_dim)(a)\n",
    "    a_probs = Permute((2, 1), name='attention_vec')(a)\n",
    "    output_attention_mul = Add()([inputs, a_probs])\n",
    "    return output_attention_mul\n",
    "\n",
    "def model_attention_applied_after_lstm():\n",
    "    inputs = Input(shape=(TIME_STEPS, INPUT_DIM,))\n",
    "    lstm_units = 32\n",
    "    lstm_out = LSTM(lstm_units, return_sequences=True)(inputs)\n",
    "    attention_mul = attention_3d_block(lstm_out)\n",
    "    attention_mul = Flatten()(attention_mul)\n",
    "    output = Dense(1, activation='sigmoid')(attention_mul)\n",
    "    model = Model(inputs, output)\n",
    "    return model\n",
    "\n",
    "model = model_attention_applied_after_lstm()\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'model_fc' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-35-9e31991649b8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;31m#The LSTM  model -  output_shape = (batch, step, hidden)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0mmodel1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSequential\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0mmodel_fc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mLSTM\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m64\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_sequences\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseq_length\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m8\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m \u001b[0mmodel1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mLSTM\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_dim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minput_dim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_dim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mhidden\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_length\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_sequences\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'model_fc' is not defined"
     ]
    }
   ],
   "source": [
    "#from keras.layers.core import*\n",
    "#from keras.models import Sequential\n",
    "\n",
    "input_dim = 32\n",
    "hidden = 32\n",
    "\n",
    "#The LSTM  model -  output_shape = (batch, step, hidden)\n",
    "model1 = Sequential()\n",
    "model_fc.add(LSTM(64, return_sequences=True, input_shape=(seq_length, 8)))\n",
    "model1.add(LSTM(input_dim=input_dim, output_dim=hidden, input_length=step, return_sequences=True))\n",
    "\n",
    "#The weight model  - actual output shape  = (batch, step)\n",
    "# after reshape : output_shape = (batch, step,  hidden)\n",
    "model2 = Sequential()\n",
    "model2.add(Dense(input_dim=input_dim, output_dim=step))\n",
    "model2.add(Activation('softmax')) # Learn a probability distribution over each  step.\n",
    "#Reshape to match LSTM's output shape, so that we can do element-wise multiplication.\n",
    "model2.add(RepeatVector(hidden))\n",
    "model2.add(Permute(2, 1))\n",
    "\n",
    "#The final model which gives the weighted sum:\n",
    "model = Sequential()\n",
    "model.add(Merge([model1, model2], 'mul'))  # Multiply each element with corresponding weight a[i][j][k] * b[i][j]\n",
    "model.add(TimeDistributedMerge('sum')) # Sum the weighted elements.\n",
    "\n",
    "model.compile(loss='mse', optimizer='sgd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /nfs/2018/j/jcruz-y-/miniconda3/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'merge' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-27-17386dc19ec5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0mattention\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRepeatVector\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mattention\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0mattention\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPermute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mattention\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m \u001b[0msent_representation\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmerge\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlstm_1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattention\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m \u001b[0mlstm_out\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDense\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m9\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mactivation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"lstm_out\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msent_representation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'merge' is not defined"
     ]
    }
   ],
   "source": [
    "#lstm layers\n",
    "inputs = Input(shape=(None, 8), name=\"inputs\")\n",
    "\n",
    "lstm_0 = LSTM(100, return_sequences=True, name=\"lstm_0\")(inputs)\n",
    "do = Dropout(0.4)(lstm_0)\n",
    "lstm_1 = LSTM(100, return_sequences=False, name=\"lstm_1\")(do)\n",
    "do_2 = Dropout(0.4)(lstm_0)\n",
    "lstm_1 = LSTM(100, return_sequences=False, name=\"lstm_1\")(do_2)\n",
    "\n",
    "attention = Dense(1, activation='tanh')(lstm_1)\n",
    "attention = Flatten()(attention)\n",
    "attention = Activation('softmax')(attention)\n",
    "attention = RepeatVector(100)(attention)\n",
    "attention = Permute([2, 1])(attention)\n",
    "sent_representation = merge([lstm_1, attention])\n",
    "attention = Activation('softmax')(attention)\n",
    "lstm_out = Dense(9, activation=None, name=\"lstm_out\")(sent_representation)\n",
    "\n",
    "#decoder layers\n",
    "decoder_0 = decoder.get_layer(\"decoder_0\")(lstm_out)\n",
    "decoder_0.trainable = False\n",
    "decoder_1 = decoder.get_layer(\"decoder_1\")(decoder_0)\n",
    "decoder_1.trainable = False\n",
    "decoder_output = decoder.get_layer(\"decoder_output\")(decoder_1)\n",
    "decoder_output.trainable = False\n",
    "\n",
    "model = Model(inputs, decoder_output, name=\"model_v1\")\n",
    "optimizer = tf.keras.optimizers.Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.00, amsgrad=False)\n",
    "model.compile(optimizer=optimizer, loss='mse')\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LSTM model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "inputs (InputLayer)          (None, None, 8)           0         \n",
      "_________________________________________________________________\n",
      "lstm_0 (LSTM)                (None, None, 100)         43600     \n",
      "_________________________________________________________________\n",
      "dropout_8 (Dropout)          (None, None, 100)         0         \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 100)               80400     \n",
      "_________________________________________________________________\n",
      "lstm_out (Dense)             (None, 9)                 909       \n",
      "_________________________________________________________________\n",
      "decoder_0 (Dense)            (None, 32)                320       \n",
      "_________________________________________________________________\n",
      "decoder_1 (Dense)            (None, 16)                528       \n",
      "_________________________________________________________________\n",
      "decoder_output (Dense)       (None, 63)                1071      \n",
      "=================================================================\n",
      "Total params: 126,828\n",
      "Trainable params: 126,828\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "#lstm layers\n",
    "inputs = Input(shape=(None, 8), name=\"inputs\")\n",
    "lstm_0 = LSTM(100, return_sequences=True, name=\"lstm_0\")(inputs)\n",
    "do = Dropout(0.4)(lstm_0)\n",
    "lstm_1 = LSTM(100, return_sequences=False, name=\"lstm_1\")(do)\n",
    "do_2 = Dropout(0.4)(lstm_0)\n",
    "lstm_1 = LSTM(100, return_sequences=False, name=\"lstm_1\")(do_2)\n",
    "lstm_out = Dense(9, activation=None, name=\"lstm_out\")(lstm_1)\n",
    "\n",
    "#decoder layers\n",
    "decoder_0 = decoder.get_layer(\"decoder_0\")(lstm_out)\n",
    "decoder_0.trainable = False\n",
    "decoder_1 = decoder.get_layer(\"decoder_1\")(decoder_0)\n",
    "decoder_1.trainable = False\n",
    "decoder_output = decoder.get_layer(\"decoder_output\")(decoder_1)\n",
    "decoder_output.trainable = False\n",
    "\n",
    "model = Model(inputs, decoder_output, name=\"model_v1\")\n",
    "optimizer = tf.keras.optimizers.Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.00, amsgrad=False)\n",
    "model.compile(optimizer=optimizer, loss='mse')\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "388658/388658 [==============================] - 164s 422us/sample - loss: 731.0006\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "731.000609906823"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(features, labels, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 310926 samples, validate on 77732 samples\n",
      "Epoch 1/4\n",
      "310926/310926 [==============================] - 588s 2ms/sample - loss: 203.7043 - val_loss: 598.5922\n",
      "Epoch 2/4\n",
      "310926/310926 [==============================] - 582s 2ms/sample - loss: 154.3962 - val_loss: 596.6076\n",
      "Epoch 3/4\n",
      "140448/310926 [============>.................] - ETA: 5:01 - loss: 140.8171"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-33-d38279c314b1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mhistory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mseq_length\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_split\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m    878\u001b[0m           \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    879\u001b[0m           \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 880\u001b[0;31m           validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m    881\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    882\u001b[0m   def evaluate(self,\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mmodel_iteration\u001b[0;34m(model, inputs, targets, sample_weights, batch_size, epochs, verbose, callbacks, val_inputs, val_targets, val_sample_weights, shuffle, initial_epoch, steps_per_epoch, validation_steps, mode, validation_in_fit, **kwargs)\u001b[0m\n\u001b[1;32m    327\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    328\u001b[0m         \u001b[0;31m# Get outputs.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 329\u001b[0;31m         \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    330\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    331\u001b[0m           \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/tensorflow/python/keras/backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   3074\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3075\u001b[0m     fetched = self._callable_fn(*array_vals,\n\u001b[0;32m-> 3076\u001b[0;31m                                 run_metadata=self.run_metadata)\n\u001b[0m\u001b[1;32m   3077\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_fetch_callbacks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3078\u001b[0m     return nest.pack_sequence_as(self._outputs_structure,\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1437\u001b[0m           ret = tf_session.TF_SessionRunCallable(\n\u001b[1;32m   1438\u001b[0m               \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1439\u001b[0;31m               run_metadata_ptr)\n\u001b[0m\u001b[1;32m   1440\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1441\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "history = model.fit(features, labels, batch_size=seq_length, epochs=4, verbose=1, validation_split=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('AE_jose_mega_val_loss_600_model.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize Model Error"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot the validation loss over the training epochs:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x19bef4d68>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAfbUlEQVR4nO3deXDc5Z3n8fdX3S21ZOuwrcM6bMtcJmCwbBzunZiYcJhgu2pJoGozAw6sK5tJQjKbnbCpmiRsZmehZmeyJKaGuIJTZCoLZJOADOFYh8AmGRKDgfaBj2AMxDpsybIt+dCtZ//4/Wy17pbUUnf//HlVdel3PK1+Hhp/fs/veZ5umXMOEREJlqxUV0BERJJP4S4iEkAKdxGRAFK4i4gEkMJdRCSAwqmuAEBxcbGrrq5OdTVERDLKW2+9dcQ5VzLcubQI9+rqarZt25bqaoiIZBQz+2ikcxqWEREJIIW7iEgAKdxFRAJI4S4iEkAKdxGRAFK4i4gEkMJdRCSA0mKd+4T9+Y/wwW9h1kKYvdD7mTcbzFJdMxGRlMrscD+4FV797wOP5RT0B/3gnwWVkKWbFREJvswO9+vuhyvXw7GP4NgHcPQD/+cBOLQT9v4K+rr7y4eyoWiBF/azzxsY/LMWQDgndW0REUmizA53gEgulF7sPQbr64XWukHB7z8+eh26TsYVNq9nP3shzKr2wj++5x8tnK4WiYhMWuaH+2iyQl6PfNYCOG/FwHPOwakjQ4P/2Afwp5fgVPPA8rmzB4Z9fM9/ZpnG+UUkrQQ73EdjBjNLvMe8K4ee7zwBxz4cGvx1b8C7vwTX1182kuf19s8O81T39/wL50EoMk2NEhHxnLvhPpacfJh7mfcYrKcLWg8ODf6jB+D9V6Cno7+shaBo3vATvLMXQvaM6WuTiJwzFO4TEc6GOed7j8H6+uDkoWGC/wN49xloPzaw/MyykYM/b46Ge0RkQhTuyZaVBQUV3qP6uqHn248NE/wfeuv1tz85sGxOgT/EM0zwF1R6cwoiIsNQuE+33FlQOQsqlw09190+zLLOD+Dwu7D3hZGXdQ4O/qIFEIlOX5sktZyD3i7oPOmtAOs65T9OxG2f9FaP5RR4K78GP7Jn6jMgAaNwTycTWdZ57AP46A/eP+SzBi/rHHQByC2arhbJcHq6/BA+OTB8O0/2bw/3c0h4x+339UyuTpY1QvAXDdof6eKQr4tDmkko3M2sCPgRsBhwwOeBfcDTQDXwIfBZ59wxMzPgEWAVcBq4xzn3dtJrfq4Za1nn6RZvQnfIss6X4VTTwPK5s/ywP29o8OfP1Th/vN7uQeE6OHTjwrYzvqc8uOccdz7+DmwskRmQM9ObeM+e4fWw8+ZA0XxvOzv+fPzPuOfk+PuWBZ1t0NGa2OPogf7tAZ8JGY4NCv7BFwX/MdKdQ06BLg5JlmjP/RHgJefcHWaWDeQB3wRecc49ZGYPAA8A3wBuBS70H1cB/+L/lKliBjOKvcewyzpP+ss6Dwxa1vnm0GWd4dz+3v7s8wb2/Ivmp/eyzt6eUYYlBofvaD3nuPO9XYm/fiRvaLhGi6CwamDYDgnf/OHPRfKSH3gziif2vN6e8V0YOlq9/886Wr3ndbaN8QI2cvAncueQU6A5qEHMOTd6AbNCIAac5+IKm9k+YIVzrtHMyoHXnHOLzOyH/vaTg8uN9BrLly93+gPZKdLbDcf/PPDTu2cvAB9CT3t/WQt5QXW2tx/X859V7fUQE9XXm9xhia5TA5egjiWcO7BXO2LwDj4/OIj9MpE8hcto+nrHf3HoaIUO/zmdrWO/RkIXh1HuHjLw/TOzt5xzy4c7l0jPfSHQDPzYzJYAbwH3A2VxgX0IKPO3K4GDcc+v848NCHczWw+sB5g/f35iLZHkC0VGXtbpHJw41L+GPz74dz87dFnnjNL+cX4Ljd5zjr9ojCUcHT5cZ5aO3Os9sz/skMWMjPyHnNGyQt5wYO6siT2/r9f7/2c8F4bjB6FzV/9FgtE7smTnJ3ZxGHL3UORdMELpNYWZSG3CwDLgy865rWb2CN4QzFnOOWdmY/yXG8g5txHYCF7PfTzPlWliBgXl3mPBtUPPtx8f+iGuox9639uDDRwPnlEyfMiO1XPOnpl2/2gkBbJC3kKAiS4G6OvzOhvjuTi01UPT7v79MS8OMyd255A/d0o+zJjIv5o6oM45t9Xf/zleuB82s/K4YZkzs3b1wLy451f5xyRocosgdylULE11TURGl5XVH6YT0dfnDQEmdGE47g1BnWiE5r39x+PntuKt+p9w5X+ceNtGMGa4O+cOmdlBM1vknNsHrAR2+4+7gYf8n7X+UzYDXzKzp/AmUltHG28XEUl7WVn+cEwBA/uuCXJu5ItD5RVJry4kvlrmy8BP/ZUyB4B1eH+i72dmdi/wEfBZv+wLeMsg9+MthVyX1BqLiGQaM29+KCffW5QwDRIKd+dcDBhuRnblMGUd8NeTrJeIiEyCPjUgIhJACncRkQBSuIuIBJDCXUQkgBTuIiIBpHAXEQkghbuISAAp3EVEAkjhLiISQAp3EZEAUriLiASQwl1EJIAU7iIiAaRwFxEJIIW7iEgAKdxFRAJI4S4iEkAKdxGRAFK4i4gEkMJdRCSAFO4iIgGkcBcRCSCFu4hIACncRUQCSOEuIhJACncRkQBSuIuIBJDCXUQkgBTuIiIBpHAXEQkghbuISAAlFO5m9qGZ7TSzmJlt84/NNrMtZvae/3OWf9zM7Ptmtt/MdpjZsqlsgIiIDDWenvsNzrka59xyf/8B4BXn3IXAK/4+wK3Ahf5jPfAvyaqsiIgkZjLDMmuAJ/ztJ4C1ccd/4jx/BIrMrHwSryMiIuOUaLg74P+a2Vtmtt4/Vuaca/S3DwFl/nYlcDDuuXX+sQHMbL2ZbTOzbc3NzROouoiIjCScYLnrnXP1ZlYKbDGzvfEnnXPOzNx4Xtg5txHYCLB8+fJxPVdEREaXUM/dOVfv/2wCngGuBA6fGW7xfzb5xeuBeXFPr/KPiYjINBkz3M1shpnln9kGbgJ2AZuBu/1idwO1/vZm4K/8VTNXA61xwzciIjINEhmWKQOeMbMz5f+3c+4lM3sT+JmZ3Qt8BHzWL/8CsArYD5wG1iW91iIiMqoxw905dwBYMszxFmDlMMcd8NdJqZ2IiEyIPqEqIhJACncRkQBSuIuIBJDCXUQkgBTuIiIBpHAXEQkghbuISAAp3EVEAkjhLiISQAp3EZEAUriLiASQwl1EJIAU7iIiAaRwFxEJIIW7iEgAKdxFRAJI4S4iEkAKdxGRAFK4i4gEkMJdRCSAFO4iIgGkcBcRCSCFu4hIACncRUQCSOEuIhJACncRkQBSuIuIBJDCXUQkgBTuIiIBpHAXEQmghMPdzEJm9o6ZPe/vLzSzrWa238yeNrNs/3iOv7/fP189NVUXEZGRjKfnfj+wJ27/YeB7zrkLgGPAvf7xe4Fj/vHv+eVERGQaJRTuZlYF3Ab8yN834JPAz/0iTwBr/e01/j7++ZV+eRERmSaJ9tz/F/C3QJ+/Pwc47pzr8ffrgEp/uxI4COCfb/XLD2Bm681sm5lta25unmD1RURkOGOGu5l9Gmhyzr2VzBd2zm10zi13zi0vKSlJ5q8WETnnhRMocx2w2sxWAVGgAHgEKDKzsN87rwLq/fL1wDygzszCQCHQkvSai4jIiMbsuTvn/qtzrso5Vw3cBfzGOfcfgFeBO/xidwO1/vZmfx///G+ccy6ptRYRkVFNZp37N4C/MbP9eGPqj/vHHwfm+Mf/BnhgclUUEZHxSmRY5izn3GvAa/72AeDKYcp0AJ9JQt1ERGSC9AlVEZEAUriLiASQwl1EJIAU7iIiAaRwFxEJIIW7iEgAKdxFRAJI4S4iEkAKdxGRAFK4i4gEkMJdRCSAFO4iIgGkcBcRCSCFu4hIACncRUQCSOEuIhJACncRkQBSuIuIBJDCXUQkgBTuIiIBpHAXEQkghbuISAAp3EVEAkjhLiISQAp3EZEAUriLiASQwl1EJIAU7iIiAaRwFxEJIIW7iEgAKdxFRAJozHA3s6iZvWFm283sXTN70D++0My2mtl+M3vazLL94zn+/n7/fPXUNkFERAZLpOfeCXzSObcEqAFuMbOrgYeB7znnLgCOAff65e8FjvnHv+eXExGRaTRmuDvPSX834j8c8Eng5/7xJ4C1/vYafx///Eozs6TVWERExpTQmLuZhcwsBjQBW4D3gePOuR6/SB1Q6W9XAgcB/POtwJxhfud6M9tmZtuam5sn1woRERkgoXB3zvU652qAKuBK4OLJvrBzbqNzbrlzbnlJSclkf52IiMQZ12oZ59xx4FXgGqDIzML+qSqg3t+uB+YB+OcLgZak1FZERBKSyGqZEjMr8rdzgU8Be/BC/g6/2N1Arb+92d/HP/8b55xLZqVFRGR04bGLUA48YWYhvIvBz5xzz5vZbuApM/t74B3gcb/848C/mtl+4Chw1xTUW0RERjFmuDvndgBLhzl+AG/8ffDxDuAzSamdiIhMiD6hKiISQAp3EZEAUriLiASQwl1EJIAU7iIiAaRwFxEJIIW7iEgAKdxFRAIoo8P9QPNJNm9voL2rN9VVERFJK4l8/UDaejbWwPdfeY8Z2SFuvnQuq2squP6CYsKhjL5miYhMWkaH+/0rL+Sa8+ZQG6vnhZ2N/PKdeubMyObTl5ezuqaSZfOL0N8JEZFzkaXDFzYuX77cbdu2bVK/o7Onl9f2NbM51sCv9xyms6ePebNzWbOkkrVLK7igND9JtRURSQ9m9pZzbvmw54IS7vFOdHTz8ruHqY3V82/7j9Dn4JLyAtYureD2JRWUF+Ym7bVERFLlnAv3eE0nOvjVjkaejTWw/eBxzOCqhbNZU1PJqsXlFOZFpuR1RUSm2jkd7vE+PHKK2lgDtbF6Dhw5RSRkrFhUytqaSlZ+rJRoJDTldRARSRaF+yDOOXbVt1Ebq2fz9gaaTnQyMyfMTZeWsbamkmvPn6MVNyKS9hTuo+jtc2w90EJtrIEXdjVyoqOH4pk5fPryctbUVFAzTytuRCQ9KdwT1NHtrbipjdXzyt4munr6WDAnjzVLKliztJLzS2amuooiImcp3CegraObl3YdojZWz+vvt+AcLK4sYG1NJZ++vIK5hdFUV1FEznEK90lqauvguR2N1Mbq2VHXihlcc94c1tRUcMvicgpzteJGRKafwj2JDjSfPLvi5sOW02SHsrjh4hLW1lRyw8VacSMi00fhPgWcc+yoa6U21sBzOxpoPtFJfk6YmxfPZW1NJdecP4dQliZiRWTqKNynWG+f4w/vt/BsrJ6Xdh3iZGcPJfk53H55BWtqKri8qlArbkQk6RTu06iju5dX9zbxbKyeV/c209Xbx8LiGaxeUsHapZUsLJ6R6iqKSEAo3FOktb2bl3Y1Uhtr4A8HvBU3l1cVsqamktsvL6e0QCtuRGTiFO5p4FBrB8/vaODZWD276tvIMrj2/GJW11Rwy+K5FES14kZExkfhnmb2N51kc6ye2u0NfNRymuxwFisvLmVNTQUrFmnFjYgkRuGeppxzxA4epzbWwPM7Gjhysov8aJhb/RU3V52nFTciMjKFewbo6e3j9fe977h5+V1vxU1ZwZkVN5UsrizQihsRGUDhnmE6unt5ZY+34ua1fU109zrOK5nBmiWVrKmpoForbkSESYa7mc0DfgKUAQ7Y6Jx7xMxmA08D1cCHwGedc8fM614+AqwCTgP3OOfeHu01FO4jaz3dzQu7vK8+2PrBUZyDJfOKWFtTwW2Xl1OarxU3IueqyYZ7OVDunHvbzPKBt4C1wD3AUefcQ2b2ADDLOfcNM1sFfBkv3K8CHnHOXTXaayjcE9PY2s5z2xt49p0Gdjd6K26uu6CYNTWV3HxpGflacSNyTknqsIyZ1QIb/McK51yjfwF4zTm3yMx+6G8/6Zffd6bcSL9zuHDv7u6mrq6Ojo6OcdUvk0WjUaqqqohExg7p9w6f8L7jZns9B4+2kxPO4saPlbG6poIVi0rICWvFjUjQjRbu4XH+ompgKbAVKIsL7EN4wzYAlcDBuKfV+cdGDPfh1NXVkZ+fT3V19Tkxkeico6Wlhbq6OhYuXDhm+QvL8vn6zYv4zzddxNt/Ps7mWD3P72jkVzsbKYiGWXVZOWtqKrlq4WyytOJG5JyTcLib2UzgF8BXnXNt8YHrnHNmNq5bADNbD6wHmD9//pDzHR0d50ywA5gZc+bMobm5edzPu2LBLK5YMIu/+/Ql/H7/ETbHGnhuewNPvXmQuQVRVtdUsHpJBZdWaMWNyLkioXA3swhesP/UOfdL//BhMyuPG5Zp8o/XA/Pinl7lHxvAObcR2AjesMwIr5tQI4Jisu0Nh7JYsaiUFYtKae/q5dd7DlMbq2fT7z9g428PcEHpTO+vStVUMn9OXpJqLSLpaMxw91e/PA7scc79c9ypzcDdwEP+z9q4418ys6fwJlRbRxtvl6mRmx3i9iUV3L6kgmOnuvwVNw3805Y/8U9b/sTS+UWsranktsvLKZ6Zk+rqikiSJbJa5nrgd8BOoM8//E28cfefAfOBj/CWQh71LwYbgFvwlkKuc86NuhRmuAnVPXv28LGPfWzcDUqWlpYWVq5cCcChQ4cIhUKUlJQA8MYbb5CdnT3m71i3bh0PPPAAixYtSvh1p7rd9ce9FTe1sQb2NLYRyjKuu6CYtTUV3HTpXGbmjGsaRkRSKCM/xJTqcI/3ne98h5kzZ/L1r399wHHnHM45srKykvZa09nuPx0+QW2sntpYA3XH2olGvBU3a2sq+YuLSsgOJ69dIpJ8SVstkyoPPvcuuxvakvo7L6ko4Nu3Xzru5+3fv5/Vq1ezdOlS3nnnHbZs2cKDDz7I22+/TXt7O3feeSff+ta3ALj++uvZsGEDixcvpri4mC984Qu8+OKL5OXlUVtbS2lpaVLbNF4XleXzX26+mK/ftIi3/3yMZ99p4Fc7G3l+RyOFuRFWXVbO2poKPl6tFTcimUZdswnYu3cvX/va19i9ezeVlZU89NBDbNu2je3bt7NlyxZ279495Dmtra184hOfYPv27VxzzTVs2rQpBTUfnrfiZjbfXbuYrd9cyY/v+Tg3LCqhNlbPnRv/yPUP/4b/8eIedje0kQ53eiIytozouU+khz2Vzj//fJYv778TevLJJ3n88cfp6emhoaGB3bt3c8kllwx4Tm5uLrfeeisAV1xxBb/73e+mtc6JioSyuOHiUm64uJTTXT1s2X2Y2lgDj//uA374/w5wUdlM1tRUsnpJBfNma8WNSLrKiHBPNzNm9H9x13vvvccjjzzCG2+8QVFREZ/73OeG/VRt/ARsKBSip6dnWuo6GXnZYdbUVLKmppKjp7r41c5GNsfq+ceX9/GPL+/jigWzWFtTwarLypmjFTciaUXDMpPU1tZGfn4+BQUFNDY28vLLL6e6SlNi9oxs/vLqBfyfL1zL779xA397yyJOdvTwd7XvcuU/vMK6H7/Bs+/Uc6oz/S9aIucC9dwnadmyZVxyySVcfPHFLFiwgOuuuy7VVZpyVbPy+OKKC/jiigvYe6iN2lgDm2MNfPXpGLmREJ+6pIzLqwopK4hSVhBlbkGU0oIc/YUpkWmkpZBpJlPb3dfneOvPx3j2nXpe2NnIsdPdQ8oU5kYoK8g5G/oDt7394pk5REK6oRRJRMYvhZT0l5VlfLx6Nh+vns3fr11MW0cPh9s6/Edn3La3/37TEZpOdNLTN7BzYQZzZuQwtzCHsvwopX7oz/UvAKX+BWF2XraWZ4qMQuEuSWdmFOZGKMyNcFFZ/ojl+vocLae6ONzWQdOJDg61dp7dPtzWSWNrB9vrjnPkZNeQ50ZCRmm+H/b5UeYW9m+XFUSZW5hDaUGU/JzwOfcdRSKgcJcUysoySvJzKMnPAQpHLNfV08eRk50cauugye/5H/LvApraOnm/+ST/9v4RTnQMnczNjYTGHAoqK4hqPkACR+EuaS87nEVFUS4VRbmjljvd1UOTPwR0yA/+w20dHD7h/dxed5xDrR109vQNeW78fEBpvtfzP7NdVpDD3MKo5gMkoyjcJTDyssNUF4dH/QPizrlJzwecmQMojev5z9V8gKQZhbucU1IxH1BWkENZYfTsfMCZfc0HyFRSuI8gGV/5C7Bp0yZWrVrF3Llzp6yuknyTmQ+IHxZ6v/kkr79/hLZR5gNK/Z7/2WGhQfuaD5CJULiPYM6cOcRiMWDkr/xNxKZNm1i2bJnCPaAmMh9w+EQnh1s7hswHHG7roKN76HxAQTTM3MLokDmAM9tlBVFK8jUfIANlRri/+AAc2pnc3zn3Mrj1oQk99YknnuDRRx+lq6uLa6+9lg0bNtDX18e6deuIxWI451i/fj1lZWXEYjHuvPNOcnNzx9Xjl2AZz3xAk9/zP3MXEL+fyHzA0JVBOZTmexeA3OwQuZGQLgTngMwI9zSya9cunnnmGV5//XXC4TDr16/nqaee4vzzz+fIkSPs3OldhI4fP05RURE/+MEP2LBhAzU1NSmuuaS7+PmAC8cxH3C4rZNDrf3bh9s62DHCfMAZkZCRGwmRlx0+G/h52SFys/2fkRC52WHysvuP95cJk+dvR/3zeZHwgOdqQjn1MiPcJ9jDngq//vWvefPNN89+5W97ezvz5s3j5ptvZt++fXzlK1/htttu46abbkpxTSWoEp0P6O7to/lE/2qgllNdtHf10t7Vy+lu/2dXD6e7euno7uV0Vy8nOrzho9PdPbR39dHe1cPp7l7G+y0lOeEs/8LQf/E4E/7eBSBMbnaWdz7SfzzqX3CGXlD6j+eEszQRnYDMCPc04pzj85//PN/97neHnNuxYwcvvvgijz76KL/4xS/YuHFjCmoo4omEEpsPGItzjs6ePk539dLe3esFfpd3MWj3j3nbPXFlevvLdPec3T96qou6Y/HP6xl2nmE0ZgwM/UjYu4MY793H4IuOf/cRlD8vqXAfpxtvvJE77riD+++/n+LiYlpaWjh16hS5ublEo1E+85nPcOGFF3LfffcBkJ+fz4kTJ1Jca5GJMzOikdCUrdrp63NnLxBn7iBOd/VfENrj7zL87ZHuPg61dcddWLwLR1fv+C4e4SwbEPrDXgjO3mWEBtx99F9ABg5Txd99hKZpyErhPk6XXXYZ3/72t7nxxhvp6+sjEonw2GOPEQqFuPfee3HOYWY8/PDDAKxbt4777rtPE6oiI8jKMmbkhJmRMzVx1N3bN+Buot2/mxjt7mPghcYrf6qrhyMnO+PKeheQvnEOWWWHswYE/ldvvIjVSyqS3m595W+aOVfbLZKJnHN09fYNHIaKG3I6e7y7l46z2wPvSu76+Dz+3YUlE3p9feWviMgUMDNywiFywiGK0uxPCgdj5kBERAZI63BPhyGj6XSutVdEpk7ahns0GqWlpeWcCTznHC0tLUSj0VRXRUQCIG3H3Kuqqqirq6O5uTnVVZk20WiUqqqqVFdDRAIgbcM9EomwcOHCVFdDRCQjpe2wjIiITJzCXUQkgBTuIiIBlBafUDWzZuCjCT69GDiSxOqkktqSfoLSDlBb0tVk2rLAOTfsx1vTItwnw8y2jfTx20yjtqSfoLQD1JZ0NVVt0bCMiEgAKdxFRAIoCOEepL+Iobakn6C0A9SWdDUlbcn4MXcRERkqCD13EREZROEuIhJAGRPuZnaLme0zs/1m9sAw53PM7Gn//FYzq57+WiYmgbbcY2bNZhbzH/elop5jMbNNZtZkZrtGOG9m9n2/nTvMbNl01zFRCbRlhZm1xr0n35ruOibCzOaZ2atmttvM3jWz+4cpkxHvS4JtyZT3JWpmb5jZdr8tDw5TJrkZ5pxL+wcQAt4HzgOyge3AJYPKfBF4zN++C3g61fWeRFvuATakuq4JtOUvgGXArhHOrwJeBAy4Gtia6jpPoi0rgOdTXc8E2lEOLPO384E/DfP/V0a8Lwm2JVPeFwNm+tsRYCtw9aAySc2wTOm5Xwnsd84dcM51AU8BawaVWQM84W//HFhpZtPzZ8bHJ5G2ZATn3G+Bo6MUWQP8xHn+CBSZWfn01G58EmhLRnDONTrn3va3TwB7gMpBxTLifUmwLRnB/2990t+N+I/Bq1mSmmGZEu6VwMG4/TqGvslnyzjneoBWYM601G58EmkLwL/3b5l/bmbzpqdqSZdoWzPFNf5t9YtmdmmqKzMW/7Z+KV4vMV7GvS+jtAUy5H0xs5CZxYAmYItzbsT3JRkZlinhfq55Dqh2zl0ObKH/ai6p8zbe93gsAX4APJvi+ozKzGYCvwC+6pxrS3V9JmOMtmTM++Kc63XO1QBVwJVmtngqXy9Twr0eiO+9VvnHhi1jZmGgEGiZltqNz5htcc61OOc6/d0fAVdMU92SLZH3LSM459rO3FY7514AImZWnOJqDcvMInhh+FPn3C+HKZIx78tYbcmk9+UM59xx4FXglkGnkpphmRLubwIXmtlCM8vGm2zYPKjMZuBuf/sO4DfOn5lIM2O2ZdD452q8scZMtBn4K391xtVAq3OuMdWVmggzm3tm/NPMrsT7t5N2nQe/jo8De5xz/zxCsYx4XxJpSwa9LyVmVuRv5wKfAvYOKpbUDEvbP7MXzznXY2ZfAl7GW22yyTn3rpn9N2Cbc24z3v8E/2pm+/Emxu5KXY1HlmBbvmJmq4EevLbck7IKj8LMnsRbrVBsZnXAt/EminDOPQa8gLcyYz9wGliXmpqOLYG23AH8JzPrAdqBu9K083Ad8JfATn98F+CbwHzIuPclkbZkyvtSDjxhZiG8C9DPnHPPT2WG6esHREQCKFOGZUREZBwU7iIiAaRwFxEJIIW7iEgAKdxFRAJI4S4iEkAKdxGRAPr/YaS77WySvsMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.legend(['Train', 'Test'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plotting the errors for each field below shows that the 'y' coordinates for each figure generally have the largest error and the errors growing rapidly approaching the tips of the fingers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = model.predict(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "shape mismatch: objects cannot be broadcast to a single shape",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-36-17377843fbde>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mxticks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrotation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m90\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mylabel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Prediction Error (mm)'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mbar\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m8\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mavg_error\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m63\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0mbar\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_color\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'coral'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.7/site-packages/matplotlib/pyplot.py\u001b[0m in \u001b[0;36mbar\u001b[0;34m(x, height, width, bottom, align, data, **kwargs)\u001b[0m\n\u001b[1;32m   2432\u001b[0m     return gca().bar(\n\u001b[1;32m   2433\u001b[0m         \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwidth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwidth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbottom\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbottom\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malign\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0malign\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2434\u001b[0;31m         **({\"data\": data} if data is not None else {}), **kwargs)\n\u001b[0m\u001b[1;32m   2435\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2436\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.7/site-packages/matplotlib/__init__.py\u001b[0m in \u001b[0;36minner\u001b[0;34m(ax, data, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1587\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0minner\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0max\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1588\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1589\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0max\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msanitize_sequence\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1590\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1591\u001b[0m         \u001b[0mbound\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_sig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0max\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.7/site-packages/matplotlib/axes/_axes.py\u001b[0m in \u001b[0;36mbar\u001b[0;34m(self, x, height, width, bottom, align, **kwargs)\u001b[0m\n\u001b[1;32m   2373\u001b[0m         x, height, width, y, linewidth = np.broadcast_arrays(\n\u001b[1;32m   2374\u001b[0m             \u001b[0;31m# Make args iterable too.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2375\u001b[0;31m             np.atleast_1d(x), height, width, y, linewidth)\n\u001b[0m\u001b[1;32m   2376\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2377\u001b[0m         \u001b[0;31m# Now that units have been converted, set the tick locations.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.7/site-packages/numpy/lib/stride_tricks.py\u001b[0m in \u001b[0;36mbroadcast_arrays\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    257\u001b[0m     \u001b[0margs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_m\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msubok\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msubok\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0m_m\u001b[0m \u001b[0;32min\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    258\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 259\u001b[0;31m     \u001b[0mshape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_broadcast_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    260\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    261\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mshape\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0marray\u001b[0m \u001b[0;32min\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.7/site-packages/numpy/lib/stride_tricks.py\u001b[0m in \u001b[0;36m_broadcast_shape\u001b[0;34m(*args)\u001b[0m\n\u001b[1;32m    191\u001b[0m     \u001b[0;31m# use the old-iterator because np.nditer does not handle size 0 arrays\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    192\u001b[0m     \u001b[0;31m# consistently\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 193\u001b[0;31m     \u001b[0mb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbroadcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    194\u001b[0m     \u001b[0;31m# unfortunately, it cannot handle 32 or more arguments directly\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    195\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mpos\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m31\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: shape mismatch: objects cannot be broadcast to a single shape"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3gAAAJcCAYAAACrJAbaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAeA0lEQVR4nO3dfbDmd1nf8c9FSCSm4aGS1jbJQtAgjRoHWQGhVUSgwdrEodVCxSI6xieEWrWDtoUOtWOhhaptoGYEDVrBwDDO0kahoygWiWYRJSYQG1OQRBSUZ6iEkKt/nHvhsG723Oec/Pacvfb1mrln79/DOXuF8Efe8/09VHcHAACAk9899noAAAAA7h4CDwAAYAiBBwAAMITAAwAAGELgAQAADCHwAAAAhlgs8KrqZVX13qr6g7s4XlX1k1V1c1W9raq+fKlZAAAATgVLruD9bJJLjnP8iUkuXH0uT/KSBWcBAAAYb7HA6+43Jnn/cU65LMnLe8O1Se5bVX9rqXkAAACm28t78M5N8u5N27eu9gEAALAD99zrAdZRVZdn4zLOnHXWWQ97yEMesscTAQAA7I23vOUtf97d5xzr2F4G3m1Jzt+0fd5q31/R3VcmuTJJDh482IcPH15+OgAAgH2oqt51V8f28hLNQ0n+2eppmo9M8qHufs8ezgMAAHBSW2wFr6pekeQxSe5fVbcmeW6S05Oku/9bkmuSfF2Sm5N8PMnTl5oFAADgVLBY4HX3U7Y43km+d6m/HwAA4FSzl5doAgAAcDcSeAAAAEMIPAAAgCEEHgAAwBACDwAAYAiBBwAAMITAAwAAGELgAQAADCHwAAAAhhB4AAAAQwg8AACAIQQeAADAEAIPAABgCIEHAAAwhMADAAAYQuABAAAMIfAAAACGEHgAAABDCDwAAIAhBB4AAMAQAg8AAGAIgQcAADCEwAMAABhC4AEAAAwh8AAAAIYQeAAAAEMIPAAAgCEEHgAAwBACDwAAYAiBBwAAMITAAwAAGELgAQAADCHwAAAAhhB4AAAAQwg8AACAIQQeAADAEAIPAABgCIEHAAAwhMADAAAYQuABAAAMIfAAAACGEHgAAABDCDwAAIAhBB4AAMAQAg8AAGAIgQcAADCEwAMAABhC4AEAAAwh8AAAAIYQeAAAAEMIPAAAgCEEHgAAwBACDwAAYAiBBwAAMITAAwAAGELgAQAADCHwAAAAhhB4AAAAQwg8AACAIQQeAADAEAIPAABgCIEHAAAwhMADAAAYQuABAAAMIfAAAACGEHgAAABDCDwAAIAhBB4AAMAQAg8AAGAIgQcAADCEwAMAABhC4AEAAAwh8AAAAIYQeAAAAEMIPAAAgCEEHgAAwBACDwAAYAiBBwAAMITAAwAAGELgAQAADCHwAAAAhhB4AAAAQwg8AACAIQQeAADAEAIPAABgCIEHAAAwhMADAAAYQuABAAAMIfAAAACGEHgAAABDCDwAAIAhBB4AAMAQAg8AAGAIgQcAADCEwAMAABhC4AEAAAwh8AAAAIYQeAAAAEMIPAAAgCEEHgAAwBCLBl5VXVJVN1XVzVX17GMcP1BVb6iqt1bV26rq65acBwAAYLLFAq+qTktyRZInJrkoyVOq6qKjTvvXSa7u7ocmeXKSFy81DwAAwHRLruA9PMnN3X1Ld9+e5JVJLjvqnE5y79X3+yT5kwXnAQAAGO2eC/7uc5O8e9P2rUkecdQ5/zbJ66vq+5KcleRxC84DAAAw2l4/ZOUpSX62u89L8nVJfq6q/spMVXV5VR2uqsPve9/7TviQAAAAJ4MlA++2JOdv2j5vtW+zb09ydZJ095uT3CvJ/Y/+Rd19ZXcf7O6D55xzzkLjAgAAnNyWDLzrklxYVRdU1RnZeIjKoaPO+eMkX5skVfV3shF4lugAAAB2YLHA6+47kjwjyeuSvD0bT8u8oaqeV1WXrk77gSTfUVW/n+QVSb61u3upmQAAACZb8iEr6e5rklxz1L7nbPp+Y5JHLzkDAADAqWKvH7ICAADA3UTgAQAADCHwAAAAhhB4AAAAQwg8AACAIQQeAADAEAIPAABgCIEHAAAwhMADAAAYQuABAAAMIfAAAACGEHgAAABDCDwAAIAhBB4AAMAQAg8AAGAIgQcAADCEwAMAABhC4AEAAAwh8AAAAIYQeAAAAEMIPAAAgCEEHgAAwBACDwAAYAiBBwAAMITAAwAAGELgAQAADCHwAAAAhhB4AAAAQwg8AACAIQQeAADAEAIPAABgCIEHAAAwhMADAAAYQuABAAAMIfAAAACGEHgAAABDCDwAAIAhBB4AAMAQAg8AAGAIgQcAADCEwAMAABhC4AEAAAwh8AAAAIYQeAAAAEMIPAAAgCEEHgAAwBACDwAAYAiBBwAAMITAAwAAGELgAQAADCHwAAAAhhB4AAAAQwg8AACAIQQeAADAEAIPAABgCIEHAAAwhMADAAAYQuABAAAMIfAAAACGEHgAAABDCDwAAIAhBB4AAMAQAg8AAGAIgQcAADCEwAMAABhC4AEAAAwh8AAAAIYQeAAAAEMIPAAAgCEEHgAAwBACDwAAYAiBBwAAMITAAwAAGELgAQAADCHwAAAAhhB4AAAAQwg8AACAIQQeAADAEAIPAABgCIEHAAAwhMADAAAYQuABAAAMIfAAAACGEHgAAABDCDwAAIAhBB4AAMAQAg8AAGAIgQcAADCEwAMAABhC4AEAAAwh8AAAAIYQeAAAAEMIPAAAgCEEHgAAwBACDwAAYAiBBwAAMITAAwAAGELgAQAADCHwAAAAhhB4AAAAQywaeFV1SVXdVFU3V9Wz7+Kcb6qqG6vqhqr6hSXnAQAAmOyeS/3iqjotyRVJHp/k1iTXVdWh7r5x0zkXJvnhJI/u7g9U1d9Yah4AAIDpllzBe3iSm7v7lu6+Pckrk1x21DnfkeSK7v5AknT3execBwAAYLQlA+/cJO/etH3rat9mD07y4Kp6U1VdW1WXLDgPAADAaItdormNv//CJI9Jcl6SN1bVl3b3BzefVFWXJ7k8SQ4cOHCiZwQAADgprL2CV1X3q6ovrqoHVdU6P3dbkvM3bZ+32rfZrUkOdfcnu/v/JvnDbATfZ+nuK7v7YHcfPOecc9YdGQAA4JRy3FCrqvtU1Y9U1fVJrk3yU0muTvKuqnpVVX3NcX78uiQXVtUFVXVGkicnOXTUOb+UjdW7VNX9s3HJ5i07+icBAAA4xW11ieark7w8yd87xmWTD0vyLVX1oO5+6dE/2N13VNUzkrwuyWlJXtbdN1TV85Ic7u5Dq2NPqKobk3wqyQ9191/s/h8LAADg1FPdvdczbMvBgwf78OHDez0GAADAnqiqt3T3wWMdW/shK1V1cZIHbv6Z7n7NrqcDAADgbrFW4FXVy5JcnOSGJHeudncSgQcAALBPrLuC98juvmjRSQAAANiVdV+T8OaqEngAAAD72LoreC/PRuT9aZJPJKkk3d0XLzYZAAAA27Ju4L00ybckuT6fuQcPAACAfWTdwHvf6r11AAAA7FPrBt5bq+oXkrw2G5doJvGaBAAAgP1k3cA7Mxth94RN+7wmAQAAYB9ZK/C6++lLDwIAAMDurPui8wuSfF+SB27+me6+dJmxAAAA2K51L9H8pWw8SfO18RRNAACAfWndwPvL7v7JRScBAABgV9YNvJ+oqucmeX0++ymav7vIVAAAAGzbuoH3pdl40flj85lLNHu1DQAAwD6wbuB9Y5IHdfftSw4DAADAzt1jzfP+IMl9lxwEAACA3Vl3Be++Sd5RVdfls+/B85oEAACAfWLdwHvuolMAAACwa8cNvKqq3vAbW51z948GAADAdmx1D94bqur7qurA5p1VdUZVPbaqrkrytOXGAwAAYF1bXaJ5SZJvS/KKqrogyQeTnJmNMHx9kh/v7rcuOyIAAADrOG7gdfdfJnlxkhdX1elJ7p/k/3X3B0/EcAAAAKxv3YespLs/meQ9C84CAADALqz7HjwAAAD2OYEHAAAwxJaBV1WnVdUbTsQwAAAA7NyWgdfdn0pyZ1Xd5wTMAwAAwA6t+5CVjya5vqr+V5KPHdnZ3c9cZCoAAAC2bd3Ae83qAwAAwD61VuB191VVdUaSB6923bR6bQIAAAD7xFqBV1WPSXJVkncmqSTnV9XTuvuNy40GAADAdqx7ieYLkzyhu29Kkqp6cJJXJHnYUoMBAACwPeu+B+/0I3GXJN39h0lOX2YkAAAAdmLdFbzDVfXTSX5+tf3NSQ4vMxIAAAA7sW7gfXeS701y5LUIv5nkxYtMBAAAwI5sGXhVdVqSl3X3Nyd50fIjAQAAsBNb3oPX3Z9K8oDVaxIAAADYp9a9RPOWJG+qqkNJPnZkZ3db0QMAANgn1g28P1p97pHk7OXGAQAAYKfWvQfv7O7+wRMwDwAAADu07j14jz4BswAAALAL616i+Xur++9elc++B+81i0wFAADAtq0bePdK8hdJHrtpXycReAAAAPvEWoHX3U9fehAAAAB257j34FXV1Zu+P/+oY69faigAAAC2b6uHrFy46fvjjzp2zt08CwAAALuwVeD1Do8BAABwgm11D97nVtVDsxGCZ66+1+pz5tLDAQAAsL6tAu89SV60+v6nm74f2QYAAGCfOG7gdffXnKhBAAAA2J2t7sEDAADgJCHwAAAAhhB4AAAAQ2z1kJVPq6pzkzxg88909xuXGAoAAIDtWyvwqur5Sf5JkhuTfGq1u5MIPAAAgH1i3RW8b0jyRd39iSWHAQAAYOfWvQfvliSnLzkIAAAAu7PuCt7Hk/xeVf1qkk+v4nX3MxeZCgAAgG1bN/AOrT4AAADsU2sFXndfVVVnJHnwatdN3f3J5cYCAABgu9Z9iuZjklyV5J1JKsn5VfU0r0kAAADYP9a9RPOFSZ7Q3TclSVU9OMkrkjxsqcEAAADYnnWfonn6kbhLku7+w3iqJgAAwL6y7gre4ar66SQ/v9r+5iSHlxkJAACAnVg38L47yfcmOfJahN9M8uJFJgIAAGBH1n2K5ieSvGj1AQAAYB86buBV1dXd/U1VdX2SPvp4d1+82GQAAABsy1YreM9a/fn1Sw8CAADA7hz3KZrd/Z7V1+/p7ndt/iT5nuXHAwAAYF3rvibh8cfY98S7cxAAAAB2Z6t78L47Gyt1X1BVb9t06Owkv7XkYAAAAGzPVvfg/UKSX07yY0mevWn/R7r7/YtNBQAAwLZtdQ/eh7r7nUl+Isn7N91/d0dVPeJEDAgAAMB61r0H7yVJPrpp+6OrfQAAAOwT6wZedfen34PX3XdmzZekAwAAcGKsG3i3VNUzq+r01edZSW5ZcjAAAAC2Z93A+64kj0pyW5JbkzwiyeVLDQUAAMD2rXWZZXe/N8mTF54FAACAXdjqPXj/srtfUFX/JUkffby7n7nYZAAAAGzLVit4b1/9eXjpQQAAANid4wZed7929edVJ2YcAAAAdmqrSzRfm2NcmnlEd196t08EAADAjmx1ieZ/Wv35pCSfn+TnV9tPSfJnSw0FAADA9m11ieZvJElVvbC7D2469Nqqcl8eAADAPrLue/DOqqoHHdmoqguSnLXMSAAAAOzEWu/BS/L9SX69qm5JUkkekOQ7F5sKAACAbVv3Ree/UlUXJnnIatc7uvsTy40FAADAdq11iWZVfW6SH0ryjO7+/SQHqurrF50MAACAbVn3HryfSXJ7kq9cbd+W5EcXmQgAAIAdWTfwvqC7X5Dkk0nS3R/Pxr14AAAA7BPrBt7tVXVmVi89r6ovSOIePAAAgH1k3adoPjfJryQ5v6r+e5JHJ/nWpYYCAABg+7YMvKqqJO9I8qQkj8zGpZnP6u4/X3g2AAAAtmHLwOvurqpruvtLk/zPEzATAAAAO7DuPXi/W1VfsegkAAAA7Mq69+A9IslTq+qdST6Wjcs0u7svXmowAAAAtmfdwPv7i04BAADArh038KrqXkm+K8kXJrk+yUu7+44TMRgAAADbs9U9eFclOZiNuHtikhdu55dX1SVVdVNV3VxVzz7Oef+oqrqqDm7n9wMAAPAZW12iedHq6Zmpqpcm+Z11f3FVnZbkiiSPT3Jrkuuq6lB333jUeWcneVaS397O4AAAAHy2rVbwPnnkyw4uzXx4kpu7+5buvj3JK5Ncdozz/l2S5yf5y23+fgAAADbZKvC+rKo+vPp8JMnFR75X1Ye3+Nlzk7x70/atq32fVlVfnuT87vZ+PQAAgF067iWa3X3aUn9xVd0jyYuSfOsa516e5PIkOXDgwFIjAQAAnNTWfdH5TtyW5PxN2+et9h1xdpIvSfLrq/frPTLJoWM9aKW7r+zug9198JxzzllwZAAAgJPXkoF3XZILq+qCqjojyZOTHDpysLs/1N337+4HdvcDk1yb5NLuPrzgTAAAAGMtFnirh7I8I8nrkrw9ydXdfUNVPa+qLl3q7wUAADhVbfWahF3p7muSXHPUvufcxbmPWXIWAACA6Za8RBMAAIATSOABAAAMIfAAAACGEHgAAABDCDwAAIAhBB4AAMAQAg8AAGAIgQcAADCEwAMAABhC4AEAAAwh8AAAAIYQeAAAAEMIPAAAgCEEHgAAwBACDwAAYAiBBwAAMITAAwAAGELgAQAADCHwAAAAhhB4AAAAQwg8AACAIQQeAADAEAIPAABgCIEHAAAwhMADAAAYQuABAAAMIfAAAACGEHgAAABDCDwAAIAhBB4AAMAQAg8AAGAIgQcAADCEwAMAABhC4AEAAAwh8AAAAIYQeAAAAEMIPAAAgCEEHgAAwBACDwAAYAiBBwAAMITAAwAAGELgAQAADCHwAAAAhhB4AAAAQwg8AACAIQQeAADAEAIPAABgCIEHAAAwhMADAAAYQuABAAAMIfAAAACGEHgAAABDCDwAAIAhBB4AAMAQAg8AAGAIgQcAADCEwAMAABhC4AEAAAwh8AAAAIYQeAAAAEMIPAAAgCEEHgAAwBACDwAAYAiBBwAAMITAAwAAGELgAQAADCHwAAAAhhB4AAAAQwg8AACAIQQeAADAEAIPAABgCIEHAAAwhMADAAAYQuABAAAMIfAAAACGEHgAAABDCDwAAIAhBB4AAMAQAg8AAGAIgQcAADCEwAMAABhC4AEAAAwh8AAAAIYQeAAAAEMIPAAAgCEEHgAAwBACDwAAYAiBBwAAMITAAwAAGELgAQAADCHwAAAAhhB4AAAAQwg8AACAIQQeAADAEAIPAABgCIEHAAAwhMADAAAYQuABAAAMIfAAAACGEHgAAABDLBp4VXVJVd1UVTdX1bOPcfxfVNWNVfW2qvrVqnrAkvMAAABMtljgVdVpSa5I8sQkFyV5SlVddNRpb01ysLsvTvLqJC9Yah4AAIDpllzBe3iSm7v7lu6+Pckrk1y2+YTufkN3f3y1eW2S8xacBwAAYLQlA+/cJO/etH3rat9d+fYkv7zgPAAAAKPdc68HSJKqemqSg0m++i6OX57k8iQ5cODACZwMAADg5LHkCt5tSc7ftH3eat9nqarHJflXSS7t7k8c6xd195XdfbC7D55zzjmLDAsAAHCyWzLwrktyYVVdUFVnJHlykkObT6iqhyb5qWzE3XsXnAUAAGC8xQKvu+9I8owkr0vy9iRXd/cNVfW8qrp0ddp/TPLXkryqqn6vqg7dxa8DAABgC4veg9fd1yS55qh9z9n0/XFL/v0AAACnkkVfdA4AAMCJI/AAAACGEHgAAABDCDwAAIAhBB4AAMAQAg8AAGAIgQcAADCEwAMAABhC4AEAAAwh8AAAAIYQeAAAAEMIPAAAgCEEHgAAwBACDwAAYAiBBwAAMITAAwAAGELgAQAADCHwAAAAhhB4AAAAQwg8AACAIQQeAADAEAIPAABgCIEHAAAwhMADAAAYQuABAAAMIfAAAACGEHgAAABDCDwAAIAhBB4AAMAQAg8AAGAIgQcAADCEwAMAABhC4AEAAAwh8AAAAIYQeAAAAEMIPAAAgCEEHgAAwBACDwAAYAiBBwAAMITAAwAAGELgAQAADCHwAAAAhhB4AAAAQwg8AACAIQQeAADAEAIPAABgCIEHAAAwhMADAAAYQuABAAAMIfAAAACGEHgAAABDCDwAAIAhBB4AAMAQAg8AAGAIgQcAADCEwAMAABhC4AEAAAwh8AAAAIYQeAAAAEMIPAAAgCEEHgAAwBACDwAAYAiBBwAAMITAAwAAGELgAQAADCHwAAAAhhB4AAAAQwg8AACAIQQeAADAEAIPAABgCIEHAAAwhMADAAAYQuABAAAMIfAAAACGEHgAAABDCDwAAIAhBB4AAMAQAg8AAGAIgQcAADCEwAMAABhC4AEAAAwh8AAAAIYQeAAAAEMIPAAAgCEEHgAAwBACDwAAYAiBBwAAMITAAwAAGELgAQAADCHwAAAAhhB4AAAAQwg8AACAIQQeAADAEAIPAABgCIEHAAAwhMADAAAYQuABAAAMIfAAAACGEHgAAABDCDwAAIAhFg28qrqkqm6qqpur6tnHOP45VfWLq+O/XVUPXHIeAACAyRYLvKo6LckVSZ6Y5KIkT6mqi4467duTfKC7vzDJf07y/KXmAQAAmG7JFbyHJ7m5u2/p7tuTvDLJZUedc1mSq1bfX53ka6uqFpwJAABgrCUD79wk7960fetq3zHP6e47knwoyectOBMAAMBY99zrAdZRVZcnuXy1+dGqumkv57kL90/y53s9BAAAcLfZr/+N/4C7OrBk4N2W5PxN2+et9h3rnFur6p5J7pPkL47+Rd19ZZIrF5rzblFVh7v74F7PAQAA3D1Oxv/GX/ISzeuSXFhVF1TVGUmenOTQUeccSvK01fd/nOTXursXnAkAAGCsxVbwuvuOqnpGktclOS3Jy7r7hqp6XpLD3X0oyUuT/FxV3Zzk/dmIQAAAAHagLJjdParq8tWlpAAAwAAn43/jCzwAAIAhlrwHDwAAgBNI4AEAAAwh8AAAAJJU1eOOse9pxzp3vxJ4O1RVX1VVX7T6/uiq+sGq+gd7PRcAALBjz6mql1TVWVX1N6vqtUn+4V4PtR0esrIDVfXjSR6ejddMvC7J1yb55SRfneSt3f1DezgeAACwA1VVSX4gyXeudj2nu1+xhyNt22LvwRvu8Um+JMmZSW5Lcm53f7yq/kOStyYReAAAcPK5XzYWcv4oyXlJHlBV1SfRqphLNHemV/+S7zyyvfrzzvjfFAAATlbXJvmV7r4kyVck+dtJ3rS3I22PSzR3oKqen+RRSe6V5NeTPCQb/2f46iS3dPd37d10AADATlTVge7+46P2fVV3v3GvZtougbdDVfWV2VjJu7aqvjDJNyT54ySv7u47j//TAAAAdz+BBwAAMIT7xXahqp5UVf+nqj5UVR+uqo9U1Yf3ei4AAODUJPB25wVJLu3u+3T3vbv77O6+914PBQAAbF9V/dw6+/Yzgbc7f9bdb9/rIQAAgLvFF2/eqKrTkjxsj2bZEe/B24GqetLq6+Gq+sUkv5TkE0eOd/dr9mQwAABg26rqh5P8SJIzN91yVUluT3Llng22Ax6ysgNV9TOrr52Nf/GbdXd/2wkeCQAA2KWq+rHu/uG9nmM3rODtQHc/PUmq6qokz+ruD66275fkhXs5GwAAsGP/o6rO6u6PVdVTk3x5kp/o7nft9WDrcg/e7lx8JO6SpLs/kOShezgPAACwcy9J8vGq+rIkP5Dkj5K8fG9H2h6Btzv3WK3aJUmq6q/HqigAAJys7uiNe9guS/Jfu/uKJGfv8UzbIkZ254VJ3lxVr1ptf2OSf7+H8wAAADv3kdUDV56a5Kuq6h5JTt/jmbbFQ1Z2qaouSvLY1eavdfeNezkPAACwM1X1+Un+aZLruvs3q+pAksd090lzmabAAwAAGMIlmgAAwCmtqv53d//dqvpINl6F9ulD2XgN2r33aLRts4IHAAAwhKdoAgAAp7yqOq2q3rHXc+yWwAMAAE553f2pJDetHqxy0nIPHgAAwIb7Jbmhqn4nyceO7OzuS/dupO0ReAAAABv+zV4PsFsesgIAAJzSquqfJ/mtJL/b3Xfs9Ty7YQUPAAA41Z2X5MeTPKSqrk/ypmwE32919/v3dLJtsoIHAACQpKrOSHIwyaOSfOXq88HuvmhPB9sGK3gAAAAbzkxy7yT3WX3+JMn1ezrRNlnBAwAATmlVdWWSL07ykSS/neTaJNd29wf2dLAd8B48AADgVHcgyeck+dMktyW5NckH93SiHbKCBwAAnPKqqrKxiveo1edLkrw/yZu7+7l7Odt2CDwAAICVqjovyaOzEXlfn+Tzuvu+ezvV+gQeAABwSquqZ+YzK3efzOoVCavP9d195x6Oty2eogkAAJzqHpjkVUm+v7vfs8ez7IoVPAAAgCE8RRMAAGAIgQcAADCEwAMAABhC4AEAAAwh8AAAAIb4/7C+J2aVRh2nAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1080x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "error = labels - preds\n",
    "sq_error = error * error\n",
    "avg_error = np.mean(sq_error, axis=0)\n",
    "plt.figure(figsize=(15, 10))\n",
    "plt.xticks(rotation=90)\n",
    "plt.ylabel('Prediction Error (mm)')\n",
    "bar = plt.bar(df.columns[8:], avg_error)\n",
    "for i in range(0,63,3):\n",
    "    bar[i].set_color('coral')\n",
    "    bar[i+1].set_color('olivedrab')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM with Fully Connected Layers\n",
    "Below is a test using a model with several dense layers after the LSTM layers, instead of using the pretrained autoencoder."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 13160 samples, validate on 3290 samples\n",
      "Epoch 1/5\n",
      "13160/13160 [==============================] - 25s 2ms/sample - loss: 446.7564 - val_loss: 101.6531\n",
      "Epoch 2/5\n",
      " 6592/13160 [==============>...............] - ETA: 10s - loss: 118.3758"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-27-ce0d44249df2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0mmodel_fc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Adam'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'mse'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m \u001b[0mhistory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_fc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mseq_length\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_split\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m    878\u001b[0m           \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    879\u001b[0m           \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 880\u001b[0;31m           validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m    881\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    882\u001b[0m   def evaluate(self,\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mmodel_iteration\u001b[0;34m(model, inputs, targets, sample_weights, batch_size, epochs, verbose, callbacks, val_inputs, val_targets, val_sample_weights, shuffle, initial_epoch, steps_per_epoch, validation_steps, mode, validation_in_fit, **kwargs)\u001b[0m\n\u001b[1;32m    327\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    328\u001b[0m         \u001b[0;31m# Get outputs.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 329\u001b[0;31m         \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    330\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    331\u001b[0m           \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/tensorflow/python/keras/backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   3074\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3075\u001b[0m     fetched = self._callable_fn(*array_vals,\n\u001b[0;32m-> 3076\u001b[0;31m                                 run_metadata=self.run_metadata)\n\u001b[0m\u001b[1;32m   3077\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_fetch_callbacks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3078\u001b[0m     return nest.pack_sequence_as(self._outputs_structure,\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1437\u001b[0m           ret = tf_session.TF_SessionRunCallable(\n\u001b[1;32m   1438\u001b[0m               \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1439\u001b[0;31m               run_metadata_ptr)\n\u001b[0m\u001b[1;32m   1440\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1441\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model_fc = tf.keras.models.Sequential()\n",
    "model_fc.add(LSTM(64, return_sequences=True, input_shape=(seq_length, 8)))\n",
    "model_fc.add(Dropout(0.2))\n",
    "model_fc.add(LSTM(64))\n",
    "model_fc.add(BatchNormalization())\n",
    "model_fc.add(Dense(100, input_dim=64))\n",
    "model_fc.add(Activation('relu'))\n",
    "model_fc.add(BatchNormalization())\n",
    "model_fc.add(Dropout(0.2))\n",
    "model_fc.add(Dense(64, input_dim=64))\n",
    "model_fc.add(Activation('relu'))\n",
    "model_fc.add(Dropout(0.2))\n",
    "model_fc.add(Dense(63, input_dim=64))\n",
    "\n",
    "model_fc.compile(optimizer='Adam', loss='mse')\n",
    "\n",
    "history = model_fc.fit(features, labels, batch_size=seq_length, epochs=5, verbose=1, validation_split=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_fc.save('FC_model_jose_finger_4.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visual Model Error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = model_fc.predict(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3sAAAKkCAYAAAC50iatAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzde5xtdV0//tdbwBQ6gMqRDFBQUbPSVL56TL/lJfOW4teH10qJLI5Falkkp8cvzeobJpmppWEiYWlmpolKpuG1bx314AVSMZHwK3y9gImdOHn//P5Ya2QYZvZlZvbMnnWez8djP2bvtd9r7/fsWbP3eu31WWtVay0AAAAMy402uwEAAADWn7AHAAAwQMIeAADAAAl7AAAAAyTsAQAADJCwBwAAMEAHbnYDa3HEEUe0Y489drPbAAAA2BQXXnjh1a217cvdt6XD3rHHHps9e/ZsdhsAAACboqo+s9J9hnECAAAMkLAHAAAwQMIeAADAAAl7AAAAAyTsAQAADJCwBwAAMEDCHgAAwAAJewAAAAMk7AEAAAyQsAcAADBAwh4AAMAACXsAAAADJOwBAAAMkLAHAAAwQMIeAADAAAl7AAAAAyTsAQAADJCwBwAAMEDCHgAAwAAJewAAAAMk7AEAAAyQsAcAADBAwh4AAMAACXsAAAADdOBmNwAAALBedp21I3v3XT22btvBR+SMnbs3oKPNY8seAAAwGJMEvWnqtjJhDwAAYICEPQAAgAGaadirqsOr6vVVdUlVfaKq7l1VN6+qd1TVp/qfN+trq6peXFWXVtVFVXX3WfYGAAAwZLPesveiJG9rrd0pyV2TfCLJ6UkuaK0dn+SC/naSPDTJ8f3llCQvm3FvAAAAgzWzsFdVhyX5kSRnJ0lr7euttWuSnJjk3L7s3CSP6q+fmORVrbM7yeFVdatZ9QcAADBks9yyd1ySq5KcU1UfrqpXVNUhSY5srX2ur/l8kiP760cl+eyi+a/opwEAADClWYa9A5PcPcnLWmt3S3JtrhuymSRprbUkbZoHrapTqmpPVe256qqr1q1ZAACAIZll2LsiyRWttff3t1+fLvx9YWF4Zv/zi/39VyY5ZtH8R/fTrqe19vLW2gmttRO2b98+s+YBAAC2spmFvdba55N8tqru2E96YJKPJzkvyUn9tJOSvKm/fl6SJ/dH5dyR5CuLhnsCAAAwhQNn/PhPS/LqqrpxksuSnJwuYL6uqp6S5DNJHtfXnp/kYUkuTbKvrwUAAGAVZhr2WmsfSXLCMnc9cJnaluTUWfYDAACwv5j1efYAAADYBMIeAADAAAl7AAAAAyTsAQAADJCwBwAAMEDCHgAAwAAJewAAAAMk7AEAAAyQsAcAADBAwh4AAMAACXsAAAADJOwBAAAMkLAHAAAwQMIeAADAAAl7AAAAAyTsAQAADJCwBwAAMEDCHgAAwAAJewAAAAMk7AEAAAyQsAcAADBAwh4AAMAACXsAAAADJOwBAAAMkLAHAAAwQMIeAADAAAl7AAAAAyTsAQAADJCwBwAAMEDCHgAAwAAJewAAAAMk7AEAAAyQsAcAADBAwh4AAMAACXsAAAADJOwBAAAMkLAHAAAwQMIeAADAAAl7AAAAAyTsAQAADNCBm90AAPNh11k7snff1WPrth18RM7YuXsDOgIA1sKWPQCSZKKgN00dALC5hD0AAIABEvYAAAAGSNgDAAAYIGEPAABggIQ9AACAARL2AAAABkjYAwAAGCBhDwAAYICEPQAAgAES9gAAAAZI2AMAABggYQ8AAGCAhD0AAIABEvYAAAAGSNgDAAAYIGEPAABggIQ9AACAARL2AAAABkjYAwAAGCBhDwAAYICEPQAAgAES9gAAAAZI2AMAABggYQ8AAGCAhD0AAIABEvYAAAAGSNgDAAAYIGEPAABggIQ9AACAARL2AAAABkjYAwAAGCBhDwAAYICEPQAAgAES9gAAAAZopmGvqi6vqour6iNVtaefdvOqekdVfar/ebN+elXVi6vq0qq6qKruPsveAAAAhmwjtuzdv7X2Q621E/rbpye5oLV2fJIL+ttJ8tAkx/eXU5K8bAN6AwAAGKTNGMZ5YpJz++vnJnnUoumvap3dSQ6vqlttQn8AAABb3qzDXkvy9qq6sKpO6acd2Vr7XH/980mO7K8fleSzi+a9op8GAADAlA6c8ePft7V2ZVXdMsk7quqSxXe21lpVtWkesA+NpyTJrW996/XrFAAAYEBmumWvtXZl//OLSd6Y5J5JvrAwPLP/+cW+/Mokxyya/eh+2tLHfHlr7YTW2gnbt2+fZfsAAABb1szCXlUdUlXbFq4n+fEk/5rkvCQn9WUnJXlTf/28JE/uj8q5I8lXFg33BAAAYAqzHMZ5ZJI3VtXC87ymtfa2qvpgktdV1VOSfCbJ4/r685M8LMmlSfYlOXmGvQEAAAzazMJea+2yJHddZvqXkjxwmektyamz6gcAAGB/shmnXgAAAGDGhD0AAIABEvYAAAAGSNgDAAAYIGEPAABggIQ9AACAARL2AAAABkjYAwAAGCBhDwAAYICEPQAAgAES9gAAAAZI2AMAABggYQ8AAGCAhD0AAIABEvYAAAAGSNgDAAAYIGEPAABggIQ9AACAARL2AAAABkjYAwAAGCBhDwAAYICEPQAAgAES9gAAAAZI2AMAABggYQ8AAGCAhD0AAIABOnCzGwAAhmHXWTuyd9/VY+u2HXxEzti5ewM6Ati/2bIHAKyLSYLeNHUArI2wBwAAMEDCHgAAwAAJewAAAAMk7AEAAAyQsAcAADBATr0A8+rMk5Nrrxldc8jhyWnnbEw/AABsKbbswbwaF/QmrQEAYL8k7AEAAAyQYZwwELvO2jH2RMXbDj4iZ+zcvUEdAQCwmWzZg4EYF/QmrQEAYBhs2QMAYG5MMlIlMVoFJmHLHgAAc2PSUShGq8B4wh4AAMAACXsAAAADJOwBAAAMkLAHAAAwQMIeAADAAAl7AAAAAyTsAQAADJCwBwAAMEDCHgAAwAAJewAAAAMk7AEAAAyQsAcAADBAwh4AAMAACXsAAAADJOwBAAAMkLAHAAAwQMIeAADAAAl7AAAAAyTsAQAADJCwBwAAMEDCHgAAwAAJewAAAAMk7AEAAAyQsAcAADBAwh4AAMAACXsAAAADJOwBAAAMkLAHAAAwQMIeAADAAAl7AAAAAyTsAQAADJCwBwAAMEDCHgAAwAAJewAAAAMk7AEAAAyQsAcAADBAMw97VXVAVX24qt7S3z6uqt5fVZdW1V9X1Y376d/V3760v//YWfcGAAAwVBuxZe8ZST6x6PbvJ3lha+32Sb6c5Cn99Kck+XI//YV9HQAAAKsw07BXVUcneXiSV/S3K8kDkry+Lzk3yaP66yf2t9Pf/8C+HgAAgCnNesveHyX59STf7m/fIsk1rbVv9revSHJUf/2oJJ9Nkv7+r/T1AAAATGlmYa+qfiLJF1trF67z455SVXuqas9VV121ng8NAAAwGLPcsnefJI+sqsuTvDbd8M0XJTm8qg7sa45OcmV//cokxyRJf/9hSb609EFbay9vrZ3QWjth+/btM2wfAABg65pZ2Gut7WqtHd1aOzbJE5K8s7X2U0neleQxfdlJSd7UXz+vv53+/ne21tqs+gMAABiyzTjP3rOSPLOqLk23T97Z/fSzk9yin/7MJKdvQm8AAACDcOD4krVrrb07ybv765cluecyNV9N8tiN6AcAAGDoNmPLHgAAADO2IVv2AACAYdp11o7s3Xf12LptBx+RM3bu3oCOWGDLHgAAsGqTBL1p6lg/wh4AAMAACXsAAAADJOwBAAAMkLAHAAAwQMIeAADAAAl7AAAAAyTsAQAADNDEJ1Wvqpsl+d4k/53k8tbat2fWFQAAAGsyMuxV1WFJTk3yxCQ3TnJVkpskObKqdid5aWvtXTPvEgAAgKmM27L3+iSvSvI/W2vXLL6jqu6R5ElVddvW2tmzahAAAIDpjQx7rbUHjbjvwiQXrntHAAAArNk0++zdJcmxi+dprb1hBj0BAACwRhOFvap6ZZK7JPlYkoUDs7Qkwh4AAMAcmnTL3o7W2p1n2gkAAADrZtLz7P1LVQl7AAAAW8SkW/ZelS7wfT7J15JUktZau8vMOgMAAGDVJg17Zyd5UpKLc90+ewAAAMypScPeVa2182baCQAAAOtm0rD34ap6TZI3pxvGmcSpFwAAAObVpGHvpulC3o8vmubUCwAAAHNqorDXWjt51o0AAACwfiY9qfpxSZ6W5NjF87TWHjmbtgAAAFiLSYdx/l26I3K+OY7GCQAAMPcmDXtfba29eKadAAAAsG4mDXsvqqrnJHl7rn80zg/NpCsAAADWZNKw94PpTqr+gFw3jLP1twEAAJgzk4a9xya5bWvt67NsBgAAgPVxownr/jXJ4bNsBAAAgPUz6Za9w5NcUlUfzPX32XPqBQAAgDk0adh7zky7ANgP7TprR/buu3pkzbaDj8gZO3dvUEcAwJCMDHtVVa3znnE1698awLCNC3qT1gAALGfcPnvvqqqnVdWtF0+sqhtX1QOq6twkJ82uPQAAAFZj3DDOhyT52SR/VVXHJbkmyU3ThcS3J/mj1tqHZ9siAAAA0xoZ9lprX03y0iQvraqDkhyR5L9ba9dsRHMAAACszqQHaElr7RtJPjfDXgAAAFgnk55nDwAAgC1E2AMAABigsWGvqg6oqndtRDMAAACsj7Fhr7X2rSTfrqrDNqAfAAAA1sGkB2j5ryQXV9U7kly7MLG19vSZdAUAAMCaTBr23tBfAAAA2AImCnuttXOr6sZJ7tBP+mR/KgYAAADm0ERhr6rul+TcJJcnqSTHVNVJrbX3zq41AAAAVmvSYZwvSPLjrbVPJklV3SHJXyW5x6waAwAAYPUmPc/eQQtBL0laa/+W5KDZtAQAAMBaTbplb09VvSLJX/a3fyrJntm0BAAAwFpNGvZ+IcmpSRZOtfC+JC+dSUcAAACs2diwV1UHJHlla+2nkvzh7FsCAABgrcbus9da+1aS2/SnXgAAAGALmHQY52VJ/k9VnZfk2oWJrTVb+gAAAObQpGHv0/3lRkm2za4dAAAA1sOk++xta6392gb0AwAAwDqYdJ+9+2xALwAAAKyTSYdxfqTfX+9vcv199t4wk64AAABYk0nD3k2SfCnJAxZNa0mEPQAAgDk0UdhrrZ0860YAAABYPyP32auq1y26/vtL7nv7rJoCAABgbcYdoOX4RdcftOS+7evcCwAAAOtkXNhrq7wPAACATTRun72Dq+pu6ULhTfvr1V9uOuvmAAAAWJ1xYe9zSf6wv/75RdcXbgMAADCHRoa91tr9N6oRAAAA1s+4ffYAAADYgoQ9AACAARL2AAAABmjcAVq+o6qOSnKbxfO01t47i6YAAABYm4nCXlX9fpLHJ/l4km/1k1sSYQ8AAGAOTbpl71FJ7tha+9osmwEAAGB9TLrP3mVJDpplIwAAAKyfSbfs7Uvykaq6IMl3tu611p4+k64AAABYk0nD3nn9BQAAgC1gorDXWju3qm6c5A79pE+21r4xu7YAAABYi0mPxnm/JOcmuTxJJTmmqk5y6gUAAID5NOkBWl6Q5Mdbaz/aWvuRJA9O8sJRM1TVTarqA1X10ar6WFU9t59+XFW9v6ouraq/7rcYpqq+q799aX//sav/tQAAAPZvk4a9g1prn1y40Vr7t4w/OufXkjygtXbXJD+U5CFVtSPJ7yd5YWvt9km+nOQpff1Tkny5n/7Cvg4AAIBVmDTs7amqV1TV/frLnyXZM2qG1vmv/uZB/aUleUCS1/fTz013Dr8kObG/nf7+B1ZVTdgfAAAAi0wa9n4hyceTPL2/fLyfNlJVHVBVH0nyxSTvSPLpJNe01r7Zl1yR5Kj++lFJPpsk/f1fSXKLCfsDAABgkUmPxvm1JH/YXybWWvtWkh+qqsOTvDHJnabucImqOiXJKUly61vfeq0PBwAAMEgjt+xV1ev6nxdX1UVLL5M+SWvtmiTvSnLvJIdX1ULIPDrJlf31K5Mc0z/fgUkOS/KlZR7r5a21E1prJ2zfvn3SFgAAAPYr47bsPaP/+RPTPnBVbU/yjdbaNVV10yQPSnfQlXcleUyS1yY5Kcmb+lnO62//S3//O1trbdrnBQAAYMyWvdba5/qrv9ha+8ziS5JfHPPYt0ryrn4L4AeTvKO19pYkz0ryzKq6NN0+eWf39WcnuUU//ZlJTl/drwQAAMBE++yl2yr3rCXTHrrMtO9orV2U5G7LTL8syT2Xmf7VJI+dsB8A2O/tOmtH9u67emzdtoOPyBk7d29ARwDMk5Fhr6p+Id0WvNst2UdvW5J/nmVjAMBokwS9aerY//jCAIZt3Ja91yT5+yRn5PrDKve21v5jZl0BADBzvjCAYRu3z95XWmuXJ3lRkv9YtL/eN6vqXhvRIAAAANOb9KTqL0vyX4tu/1c/DQAAgDk0adirxadBaK19O5Mf3AUAAIANNmnYu6yqnl5VB/WXZyS5bJaNAQAAsHqThr2nJvnhJFcmuSLJvZKcMqumAAAAWJuJhmK21r6Y5Akz7gUAAIB1Mu48e7/eWnt+Vb0kSVt6f2vt6TPrDAAAgFUbt2XvE/3PPbNuBAAAgPUzMuy11t7c/zx3Y9oBAABgPYwbxvnmLDN8c0Fr7ZHr3hEAAABrNm4Y5x/0Px+d5HuS/GV/+4lJvjCrpgAAAFibccM435MkVfWC1toJi+56c1XZjw8AAGBOTXqevUOq6rYLN6rquCSHzKYlAAAA1mqi8+wl+ZUk766qy5JUktsk2TmzrgAAAFiTSU+q/raqOj7JnfpJl7TWvja7tgAAAFiLiYZxVtXBSU5L8kuttY8muXVV/cRMOwMAAGDVJt1n75wkX09y7/72lUl+dyYdAQAAsGaThr3btdaen+QbSdJa25du3z0AAADm0KRh7+tVddP0J1ivqtslsc8eAADAnJr0aJzPSfK2JMdU1auT3CfJz8yqKQAAANZmbNirqkpySZJHJ9mRbvjmM1prV8+4NwAAAFZpbNhrrbWqOr+19oNJ3roBPQEAALBGk+6z96Gq+h8z7QQAAIB1M+k+e/dK8tNVdXmSa9MN5WyttbvMqjEAAABWb9Kw9+CZdgEAAMC6Ghn2quomSZ6a5PZJLk5ydmvtmxvRGAAAAKs3bp+9c5OckC7oPTTJC2beEQAAAGs2bhjnnfujcKaqzk7ygdm3BAAAwFqN27L3jYUrhm8CAABsHeO27N21qv6zv15JbtrfXjga56Ez7Q4AAIBVGRn2WmsHbFQjAAAArJ9JT6oOAADAFjLpefaARXadtSN79109tm7bwUfkjJ27N6AjAAC4Plv2YBUmCXrT1AEAwHoT9gAAAAZI2AMAABggYQ8AAGCAhD0AAIABEvYAAAAGSNgDAAAYIGEPAABggIQ9AACAARL2AAAABkjYAwAAGCBhDwAAYICEPQAAgAES9gAAAAZI2AMAABggYQ8AAGCAhD0AAIABEvYAAAAGSNgDAAAYIGEPAABggIQ9AACAATpwsxuAuXDmycm114yvO+Tw5LRzZt8PAACskS17kEwW9KapAwCATSbsAQAADJCwBwAAMEDCHgAAwAAJewAAAAMk7AEAAAyQsAcAADBAwh4AAMAACXsAAAADJOwBAAAMkLAHAAAwQMIeAADAAAl7AAAAAyTsAQAADJCwBwAAMEDCHgAAwAAJewAAAAMk7AEAAAzQgZvdAAAAMLldZ+3I3n1Xj6zZdvAROWPn7g3qiHk1sy17VXVMVb2rqj5eVR+rqmf0029eVe+oqk/1P2/WT6+qenFVXVpVF1XV3WfVGwAAbFXjgt6kNQzfLIdxfjPJr7bW7pxkR5JTq+rOSU5PckFr7fgkF/S3k+ShSY7vL6ckedkMewMAABi0mYW91trnWmsf6q/vTfKJJEclOTHJuX3ZuUke1V8/McmrWmd3ksOr6laz6g8AAGDINuQALVV1bJK7JXl/kiNba5/r7/p8kiP760cl+eyi2a7opwEAADClmYe9qvruJH+b5Jdba/+5+L7WWkvSpny8U6pqT1Xtueqqq9axUwAAgOGYadirqoPSBb1Xt9be0E/+wsLwzP7nF/vpVyY5ZtHsR/fTrqe19vLW2gmttRO2b98+u+YBAAC2sFkejbOSnJ3kE621P1x013lJTuqvn5TkTYumP7k/KueOJF9ZNNwTAACAKczyPHv3SfKkJBdX1Uf6ab+R5HlJXldVT0nymSSP6+87P8nDklyaZF+Sk2fYGwAAwKDNLOy11v4pSa1w9wOXqW9JTp1VPwAAAPuTDTkaJwAAABtL2AMAABggYQ8AAGCAhD0AAIABEvYAAAAGSNgDAAAYIGEPAABggIQ9AACAARL2AAAABkjYAwAAGCBhDwAAYICEPQAAgAES9gAAAAZI2AMAABggYQ8AAGCAhD0AAIABEvYAAAAGSNgDAAAYIGEPAABggIQ9AACAARL2AAAABkjYAwAAGCBhDwAAYIAO3OwGgM1x7OlvHVtz+fMevgGdAAAwC7bsAQAADJCwBwAAMECGcQL7hV1n7cjefVePrdt28BE5Y+fuDegIAGC2bNkD9guTBL1p6gAA5p2wBwAAMEDCHgAAwAAJewAAAAMk7AEAAAyQsAcAADBAwh4AAMAACXsAAAADJOwBAAAMkLAHAAAwQMIeAADAAAl7AAAAAyTsAQAADJCwBwAAMEAHbnYDAAAAKzrz5OTaa8bXHXJ4cto5s+9nC7FlDwAAmF+TBL1p6vYjwh4AAMAACXsAAAADJOwBAAAMkLAHAAAwQMIeAADAAAl7AAAAAyTsAQAADJCwBwAAMEDCHgAAwAAduNkNAADsL3adtSN79109tm7bwUfkjJ27N6AjYMhs2QMA2CCTBL1p6gBGEfYAAAAGSNgDAAAYIGEPAABggBygBQAA2G8de/pbJ6q7/HkPn3En60/YAxiqM09Orr1mfN0hhyennTP7fgCADWUYJ8BQTRL0pqkDALYUW/YAYEKTnCPN+dEAmBe27AHAhCY595nzowEwL4Q9AACAARL2AAAABsg+ewCsypAPVQ0AQ2DLHgAAwAAJewAAAAMk7AEAAAyQsAcAADBADtACAMDsnHlycu01o2sOOTw57ZyN6Qf2I7bsAQAwO+OC3qQ1wNSEPQAAgAEyjBMAWN4kw+8SQ/DYVM75CSuzZQ8AWN6kQ+sMwQOYS8IeAADAAAl7AAAAAzSzsFdVr6yqL1bVvy6advOqekdVfar/ebN+elXVi6vq0qq6qKruPqu+AAAA9gez3LL350kesmTa6UkuaK0dn+SC/naSPDTJ8f3llCQvm2FfAAAAgzezo3G21t5bVccumXxikvv1189N8u4kz+qnv6q11pLsrqrDq+pWrbXPzao/tpZdZ+3I3n1Xj63bdvAROWPn7g3oCAAA5ttG77N35KIA9/kkR/bXj0ry2UV1V/TTIEkmCnrT1AEAwNBt2gFa+q14bdr5quqUqtpTVXuuuuqqGXQGAACw9W102PtCVd0qSfqfX+ynX5nkmEV1R/fTbqC19vLW2gmttRO2b98+02YBAAC2qo0Oe+clOam/flKSNy2a/uT+qJw7knzF/noAAACrN7MDtFTVX6U7GMsRVXVFkuckeV6S11XVU5J8Jsnj+vLzkzwsyaVJ9iU5eVZ9AQAA7A9meTTOJ65w1wOXqW1JTp1VLwAAAPubTTtACwAAALMzsy17AADjHHv6W8fWXP68h29AJwDDY8seAADAAAl7AAAAAyTsAQAADJCwBwAAMEAO0MKydp21I3v3XT2yZtvBR+SMnbs3qCMAAGAawh7LGhf0Jq0BAGCLOfPk5NprRtcccnhy2jkb0w+rZhgnAABwnXFBb9IaNp2wBwAAMECGccIGmOSkwYkTBwMAsH6EPYD1Yh8HAGCOGMYJsF7s4wAAzBFb9gBgXkyydTixhRiAiQh7ADAvJt3yawsxK/GFAbCIYZwAAEPhCwNgEWEPAABggIQ9AACAARL2AAAABkjYAwAAGCBhDwAAYICEPQAAgAES9gAAAAZI2AMAABggYQ8AAGCADtzsBgAAgP3Hsae/daK6y5/38Bl3MnzCHpvjzJOTa68ZX3fI4clp58y+HwAAGBjDONkckwS9aeoAAIDrsWVvDuw6a0f27rt6bN22g4/IGTt3b0BHAADAVmfL3hyYJOhNUwcAACDsAQAADJBhnABbxCRHL3PkMgBggS17AAAAAyTsAQAADJCwBwAAMEDCHgAAwAAJewAAAAPkaJwAAEzMkYFh67BlDwAAYICEPQAAgAES9gAAAAZI2AMAABggYQ8AAGCAhD0AAIABcuoFANhPTHLI/MRh8wGGQtgDYP915snJtdeMrjnk8OS0czamHwBYR4ZxArD/Ghf0Jq0BgDlkyx6wNU2yRSaxVQaA+ebzjBkS9oCtadKtLbbKALNkRX3wdp21I3v3XT22btvBR+SMnbunfwKfZ8yQYZwAAKtlRX3wJgl609TBRrJlj0FyxDkAAPZ3tuwBAAAMkLAHAAAwQIZxAgDAgE2ye4tdW4bJlj0AAIABsmVvf+Cw0AAAsN+xZW9/4LDQAACw3xH2AAAABkjYAwAAGCBhDwAAYIAcoAWYiV1n7cjefVePrdt28BE5Y+fuDehoOpMcpjpxqGoAYH7ZsgfMxCRBb5o6AACmI+wBAAAMkGGcs+C8dgAAwCYT9mbBee0AgHVg/2FgLYQ9AGCwJjlY1LweKApgrYQ9AJgRW2U23yQHgXKgKGCoHKAFAABggIQ9AACAATKME4C5NMm+Von9rQBgJbbsATCXJt2Pyv5WALA8W/YAAGCdODAT80TYY11M8sbmTQ0AADaOYZwAAAADZMveFmR4AENkuQYAWF/CHgAb48yTk2uvGV93yOHJaefMvh8AGLi5CntV9ZAkL0pyQJJXtNaet8ktAQusqLNWkyw/09Sxf5rkvcj7EKP4PGM/Mjdhr6oOSPInSR6U5IokH6yq81prH9/czoAkVtSB+TDJe8wa3ocMKd8P+DxjPzI3YS/JPZNc2lq7LEmq6rVJTkwi7AEwlpV0ALi+eToa51FJPrvo9hX9NDlFZmcAACAASURBVAAAAKZUrbXN7iFJUlWPSfKQ1trP9beflORerbVfWlJ3SpJT+pt3TPLJDW109Y5IcvV+VL8Rz6F+fes34jnUb/5zqF/f+o14DvWb/xzq17d+I55D/frWb8RzbPX6zXSb1tr2Ze9prc3FJcm9k/zDotu7kuza7L7W8ffbsz/Vz2NP6jf/OdRv/nOo9zcbWv089qR+859Dvb/ZZrxG83iZp2GcH0xyfFUdV1U3TvKEJOdtck8AAABb0twcoKW19s2q+qUk/5Du1AuvbK19bJPbAgAA2JLmJuwlSWvt/CTnb3YfM/Ly/ax+I55D/frWb8RzqN/851C/vvUb8RzqN/851K9v/UY8h/r1rd+I59jq9XNpbg7QAgAAwPqZp332AAAAWCfCHuuiqo5ZZtr3bEYvi55/zT31Bwta6b5brKYvVjZvy9Gsl6H+fssRW473xmGpqgOmrP+fS+epqruvb1cbb96W63nrh61J2JuRqvqxZaadNKL+BVX1/VM8/qer6qlLpr1lRP3vVNWBi24fWlXnjKi/sKpOraqbTdjSv1fVX1XVwYumjdz/ct56qqp3V9Wxi27fM91RYleyu6r+pqoeVlU1rpmqelJVbVsy7SdG1N95mWn3G1H/tClem4V5Lqiqhy2ZtuIY9Tn8m81bP9MuQ8kWX45WsQxt6X42qKenLLl9QFU9Z0T9G6rq4VU10Wd6VX2rqp63eHmrqg+NqJ+r98b+/nl7jeatn7+oqsMW3b5NVV0w4ik+VVVnLvf/sIJ/SPLOqrrlommvGDXDktqFaXccUf+ISV+fvv7sqvqhJdN+a0T9XC3X89ZPf/+8/c3m6v9sqxD2ZufZVfWyqjqkqo6sqjcnecSI+k8keXlVvb+qnrr4TXoF30hy/6o6p6775ueoEfUHJnl/Vd2lqh6U7g3kwhH1j0/yvUk+WFWvraoHj3kzuTjJ+5L8U1Xdrp827s1n3no6I8nbquoXq+p/J/nTJCePqL9Dup13n5Tug/L3quoOI+pfkuR9VfV9i6b99oj611XVs6pz06p6Sd/jSo5M99q8rqoeMsmbf5LjkjxryZvlCSPq5+1vNm/9TLsMJVt/OZp2Gdrq/WxETw+sqvOr6lbVfQm4O8m2EfUvTfKT6Zaf541aGet9LN3n/9ur6ub9tK303pjM32s0b/38U7r3xodV1c8neUeSPxpRf9ck/5bkFVW1u6pOqapDR9R/MsmZSd5TVT88QT9J9zd+3MKNqvrVJG8cUf/4dK/P86vqTmMeO0kenOTcqnryommPHFE/b8v1vPWTzN/fbN7+z7aGzT7R31Av6RaOX0vyqf7yxAnnu2OS5yX5TJLXJLn/CnUf6n/+epL3J7n1wrQRj/3AJP+d5P8luf2E/dwo3T/elUn+b5LnJrn5iH7uk+Tj6YLtyH7msack90sXpD+X5Hum+Hvfv+/nmiTvSXLvZWo+nOTu6d5MHrswbcRjHpLkj5P8S5J/TbIryY0mWO4enOS1SS5N8ntJbjei/kPpAtNLk7w5yWFbbTmaw35WtQxt1eVo2mVoCP3Muqd+nscnuTrdZ8F9Jlx+Dkvy1CSfTfLP6VYUDxqxXD++X47uMavlehbL9By/RvPWz31X+Tf70f5vdm2Sc7PM++qifo7vl+9fmqCfW/XL/98keW+Ss5J895h5Dk2yM91K/b8kOSXJthH/Z4f1z/En/f/cyOVo3pbrOexnHv9mc/V/thUum97AUC9Jbp7kdUnelm4F4vT0Rz8dMc8BSU5M8nfptk48q/8HeO0ytR9edP3HklyS5IsjHvtH+gV3V7oQ+fdJvndMP3dJ8sJ03+C9OMm9kvxqko+M6edW6baG7Bvz+HPVU5LfTLcl5979G9UlSR4+ov4WSZ6RZE+StyZ5dP9GdUKSf1+mfuFN5Ih037L+QZKLRjz+jdN9c/qRdCuTT5hw2btrum9wL0nysnRv8M9foXbxa/Qz/e9/xRb6m81bP1MtQ0NYjlaxDA2inxn3dHy6FZKz0q1g/WmSg6dYjs5Lt7LykiTvHtPPDyS5KMk167Vcz3qZntPXaN76eVK6LXVPTLfF6ENJ7jqi/oB0X4C9sV+On5luC/ZjkvzbmH4OSbe+880J/mdOTXJFui/ZfnjC/7NbJPnlJJene4//VJKnjenpt9Jt3bxsqyzX89bPnP7N5ur/bKtcNr2BoV76N9mf7a/fNN1K5T+PqH9h/89wVpJ7Lrnvk8vUP2LJ7dskefaIx/9Akjsvuv3oJJeMqL8wyQXpNn9/15L73rBM/a2W3D4wyY+MeY3mqqd0K203XfKavmPM3/g3kxy9zH3PWmbaWxddv1G6Fcxvj3j8j6YbYnFQuqDxpiR/M6L+Gf1r9A9JHpv+W6v+uT69wjw7l9y+R5JXbqG/2bz1M9UyNITlaBXL0JbuZ4N6uiTJA/vrle7LiI+NqH9jui3Pu5ZZZvcsU3+PJbcPS/Lk9VquZ71Mz+lrNG/9/F2SWy66fc+MHgFwWZKzs8zKfJIXj/pbLKq79Zj7/zHJq5IcnuQH071//8GI+oXweXGS0xZ+nyQHJ7l8mfrnLrn9iCTv3CrL9bz1M6d/s7n6P9sql01vYKiX5d70Mnol8eQkh6xw32Hr0M8By0y7xYj6227AazR3PU3Z/8gttevw+CcsM+1JI+qfm+Q2K9z3fUP8m81bP6v8Hfar5Wir97NBPR26zLQ7jKhfdrj/Zl1mvUzP42s0b/2s8Jw3HnHfyKF56/T8j1py+8Akvzmi/tyV1psWVvg3+PWb+XI9b/3M299sK/yfzePFSdUBAAAGyNE4AQAABujA8SXMo6r6i9bak8ZN24A+Hj3q/tbaGzaql6WqOzfar6YbUvvzVXV8kju21lY8H+H+qLqThN8zSUvywdba5zehh7lcjixDk5mHZWjeVdVtkhzfWvvHqrppkgNba3s3uIeRJ71urW3580mt1by+Rv3pPX4q3VD0366qW6c7WuMHNriPZ466v7X2hxvVy3KqOxXVndK9F32ytfb1zeyHradfH7lvumXon1pro041sSUIezMybRhbRXj7/iW1B6Tb6X/p4061Er2Kle5R5w5sSVZcSZ92RXoVK97npDuIwr3721emO3zw9er782qtOJ65tfb0jaxfMu9UK9GrqP+5JM9O8s50Ozu/pKp+u7X2yiV1c7kcbdVlaDXzzOtyNOkyNGtVtTfLvz6VpLXWVjxn2GqC2DTz9Oc5OyXdUZpvl+TodEeRe+CSuqmCxiqCyQtGlSd5wHJ3VNVtk7wo3f/Bt9MdTv1XWmuXbXT9HL9GU4WxVYS3l6Z7bR6Q7oBCe5P8bZL/seRxpwpjqwhvo85pNta0YWya+qp6eLr/q0+n+78/rqp2ttb+fkndXH7m9+fIe1mSI1trP1BVd0nyyNba725G/SxV1cUZ/X59lxHzThXGpqmvqpcmuX2Sv+on7ayqH2utnTrqOeadsDc7E4WxaeuraleS30hy06r6z4XJSb6e7uSaS027Ej1VfWtt3AmjR5loRXoN9bdrrT2+qp7Y97qv/4Bdas+Ufc+6Psn0K9GrXOk+LcndWmtf6h/jFukOa7x0nnldjrbqMrSaeeZ1OZpoGZo2jE1b31pb1UropEFsjfOcmi48v7/v9VNVdctl6qYNGlPVt9buP6J+lNekOwfW/+pvPyHdytC9NqF+Xl+jicLYGurv1Vq7e1V9uO/zy30QWmra/4Op6ltrz53y8b9j0jC22vp0f+v7t9Yu7ee/XbpTEiytn8vP/CR/lu799Kwkaa1dVFWvSbJSGJtZ/bRhbBXh7SdW6HGkacPYKsLbA9IdZKv185+b7vROW9uoo7e4TH9Jd3jXvUm+meQ/+8veJF9KcsZa6xfNt+J9m/i7PzzdSd6fvXAZU7+n/7n4vCYfXcf6f0532ouFc8vcLskHNvt1muL1/GQWHVky3blibnAajtXWL3qNbrzo9o0z4hQh87YcWYY2fzmax2Wo7+OWSW69cBlR95G+58XLxMVjHnuqeZK8f/Fyl+6L1rHnt5rx6/MDSR6X5MkLlxG1N+h1zP/NTOvn9DVaeI+Y9L1l2vr3pzsX3sJ82zPm5NMzfm1uku5LjJem+2LnlRlxOpF+nkuy6ATt/fvpqFPlTFv/wSW3a+m0eb4s9LpkmbjB+V83oj7daR9WvKy1fg2v0SVZdBTSdMce+cQ61r9lcb99/2/e7GVjrRdb9tZZa+2MJGdU1RmttV3rXb/IW6rqkNbatVX100nunuRFrbXPrDRD/y3Z96d7k154/t9ej/qq+tN051G5f5JXpDsR67h9Cb7eD39a+Abldkm+to71v5XupPbHVNWrk9wn3SkullVV29OdyP7Ouf7vvNKwnZnWpwv8i4eFLXwJsJJp65PuJNLvr6o3pXtdT0xy0cLQnrbM/hdzthxt6WVoNfPM4XI09TLU/x63XNL//x3xHBPXV9Uj033D/71Jvpjuw/oTWTJ6YpGvtda+vrDBtqoOzIghWKuc5z1VtTAi40FJfjHJm0c9QVX9QG74N37VetRX1XOS3K+vPz/JQ9OdzHilx//7qjo9yWvT/Z6PT3J+Vd28f57/2OD6qX/naetX8Rp9ox+Rs/Desj3dlruVTFv/4nTnDLtlVf3vdO+Nv7lScVXdJMlTcsP33p9dj/okf5FuRfrB6bZM/lS6/7NR9rZ+q1vvslz/vWat9Xuq6vx0J3hv6c6B+cF+GF/aDXc3mLfP/Kv7z6SFZeIxST638q87u/pR65LrUb+gqnakO8H596X7Au2AJNe2lYfdX5ruC7yF5zumn7aSaeu3JflEVX0g3et0z3TL1XlJ0lp75LjfaR4Je7MzbRibtv5lSe5aVXdNtw/SK9J9CP3ocsXTrkSvYqX7h1trd6mqi1prz62qF+SGQyeW+q1MsSI9bX1r7e1VdWGSHem+4XtGa+3qEY//6iR/nW7L0lOTnJTkqk2sn3YlejUr3Z/uLwve1P9cdmjPHC5Hv5WtvQytZp55W46mXYamCmOrCG+/k+7v9Y+ttbtV1f2T/PQKtckqgtgq5jk93Yr0xUl2Jjm/tfZnKxVPGzRWEUwek+Su6b7hP7mqjkzylyP6f1z/c+eS6U9It4zcdoPr5/E1miqMTVvfWnt1/170wHTvRY9qrY0KV9OGsWnrb99ae2xVndhaO7e64YDvG1GfTBnGVlF/kyRfyHXrQVelG5nxiCy/u8G8feafmm53nDtV1ZVJ/j3d32Gz6qcOY6sIb3+c7v/8b5KckG4L+h1GtDRtGJu2/tkjnnvLcp69Gamqi9J9UNwlyZ+nW9F9XGttpTA2bf2HWjd+/9lJrmytnb0wbaXHX7QSfZeq+u4kf99a+5/rVP/+1tq9qmp3kken2xLwsdba7Zd/hb4z3y1y3Yr07jEr0lPVV9UFrbWlB0C4wbRF913YWrvHwu/cT/tga23ZfSg2oP45K/1uyQ33nZi2fjXmcTnaysvQaubZ6stRVX003X4R1wtjrbWnrFP9ntbaCf18d2utfbuqPtpau+sK9TdKF8R+PN0y8Q+jgthq5qmqZ7TWXjRu2qL7Ls51QeOuC0Gjtfagdar/QGvtnn14uH+6rSWfaK3dadTvPU/m8TWqqjvlujB2wZgwNlV9TX/Qtw/3/y8L770HJXlfa23HOtUvvD7vTfdlx+fTDXG/QTBfNM85K92Xbr+u621FnLZ+WnP4mX9ca+3fq+qQJDdqre1dmLYZ9f08e7JMGGsrjERbTX3/fr34Nfpwa+1uK9Qvu068oLX2nrXUD5Ute7PzzdZaq6oTk/xxH8aWXTlZZf3e6g7W8tNJfqRf+ThoRP1/9z/3VdX3pluJvtU61r+lqg5PcmaSD6X7BuUVI+oXrzS/dZlpq66vbjjKwUmOqKqbpfsgTZJDkxw1oqVv9D8/V93Qw/+X7gAMm1I/7Ur1NPVV9UettV+uqjdnmeFnbeWhCnO1HA1gGVrNPHOxHK1hGfpGa+1LVXWjqrpRa+1dVfVHI55q2vpr+i8V3pvk1VX1xSTXjqh/Wh+6vhPWRgWxVc5zUrqjTS72M8tMW/DffUj9ZlUdmm6L5jEj+pm2fk//f/Zn6Q5Y9F/pjoB5PVX1gNbaO2uFo+u2Gw6Lm2n9EnPxGi36XRaC1yXLTFtzfaY/6NvC+8Q11Q1f/Xy6/VjXq/7l/Xvj/5fkvCTfndFbMtOmPBDXpPVV9euttefXCkfBbCsfqXiuPvPTHaDn7q21xe9Xr8/Kf+dZ1ydJWmuXVtUBrbVvJTmnuoMErbjb0ZT1+6o70NBHqur56YaVrngO8GnD2aT1VfVPrbX71g0PDDb2aM5bgbA3O9OGsWnrH5/kJ5M8pbX2+eoO23zmiPppw9hU9a213+mv/m1VvSXJTVprX1mudtoV6VWseO9M8svphn1duKj+P9MNGVjJ71bVYemGxb6kf/xf3uj6aVeiV7nS/Rf9zz8Y0e9y5mI5GtAytJp55mU5Wu0yNG0Ym7b+xCRfTfIr6YYoHZZuWNpKpg1iE89T3VFcfzLJcdUPG+ptS7Lsfmi9qYLGtPWttV/sr/5pVb0tyaGttYuWKf2RdEdlXe7oussNi5t1/WLz8hot2KgjcC+8F610BO4F04axaesvaK19Od3/5W37Xo9brnDaMLaK8LawRXTao2Au9176KxtdX90W3u9PctiSLz4OzaJ9/Taqfompwtgq6p/U3/9L6V6bY9KN6ln6O0wVxlYR3u6f7o41nVpkXhnGOSPVnafqJ9Md/eh9fRi7X1t5Z/Cp6tfY23dlRBhbTX3/QfXwJMdm0ZcIbfkDfDwj161IX5nrr0j/WWvtj9dSv2i+p7XWXjLBr7hQf5/W2v8ZN23W9VV1j9bahbXC8INlhilMVd/P8+ettZ9Zrn5Sm7kcDWUZWs0887IcrXYZqm4I0VfT/Q0WwtirW3/qhrXWL5rv0Fx/GfqPJfcvBLH75vr7Gm1L8u3ltg5PO0915+I7LskZ6fbbW7A33REovznqd+gf49iMDxpT1deEw5Or6tFjtq4tfdyZ1o94nGOzzq9RX3eX3PC9aOnWye+EsST7siSMtSXD16atXzTfVAdxq2WG5y03bQ31N9htpPphi8vUPqK19uaqOmm5x2qtnbvG+t9rrf3GcrVbQXUjuh6V5JHpgvaCvUle21r7542sXzLvbdLtB3njdGHssCR/0lr79DrVTzTEvaoOaq19IxNaRf2Ku0ENgbC3xazi24qF+SYOY6usPz/dCtnFWXREsTZiSNgqVqSnqu/nmeboa8t9eI3aD3Im9dOuRK9mpXu1b2zzthxt9WVoNfPMy3K01g/HcWFstfVVtTPJc9MtR9/Ode+Nt11SN3UQW4/wNolJw9i09XXdFu53pTtYyeIt3G9rS/ZHm/ZvPOv6ZeYfG8ZWW19Vr0y3D/3Hct17UWsrH81y2jA27RG4U92Wt+Nz/fei965QO3EYm6a+rttK9Px052xbcGiS01prNzhw0rRhbBX1q/08Oy7J03LDZWLZIegbUH/v1tqordMbWt/PM+3+xtPWL7fc3WCfvQ14L1pxP8EhMIxznU0bxqatb63dt/857abmN2eZleh1rD+63fCkmSO11l4yzYr0tPU14dHXqureSX44yfbqjzjYOzTdkaSWPu5M69OtYExj2vokObiq7pbrVvaup7X2oRXmm6vlaKsuQ6uZZw6Xo1UtQyuFsSxzxMXV1Cf5tSQ/0MYcqKd1Rzr+TJJ7j6pb6zxJUhMeoa7md3jy3FkpjGWFoZ/T1ifZ0Vq786T9tNZ2TRPGpq2vqp9L8owkR6c7z+OOdMNQH7CkbtZD/O6Y7oTYh+f6w2/3Jvn55XpP8pB0WzMnNW39AUuW/+sZ8UXS3yU5O93n1CSfT7Ou/3BVnZrJT38x6/r/v70zj5ejKvP+90kQAwhoQJyIZAFEcNh0AGVxQQd0RBRhxDfgiAgoimw68roL4sYiowIqjAEEGUZQgYAQNglbDJBAIETICAFGNDjMDEvEly087x/P6dy6fauq6znddbtu3/P7fOrTt6q/p/t039NV51fnnOcB/zT3bqe4r0P+FPfc/22JvHz7dXWYim5QjxUls9djec1YjHkLoyVL2u/CdpDXjHn5K0Vkd1W9umqBqh3pWJ7qobNXx9YnrMbwcPFPhdcYbd7biY7pdG+IhbPPK6O0dSAyalQ7GsNtKKZM09pRbBuqZMa64B/ApsdVUlUj1mWZquHFvWbMxYe76993jHBvLhYpul2tm5Htv+26+axcZiyC/62IvF5Vf1cFrmrGYvnAbo9FEN41mLRv5XBeM+biVfVS4FLnKJHXjHn5zRne/ofhFN8YekZVf1BY69Hn606XUZn3mrEI8zYPW8+3PnYdaWkFkHdO8JoxLz8Ru756TeKYUJrGWYO8ZizGvInlwDpcOyQizvAnYAuqq3aivfwHsE7wBCwCVccIRtKw0NkiMi3cuUcsQM7LVPWpkvrXwodR3tspuHBpW0JWLx/KRE1ZaFo7GuttKKZMU9pRF21oDrC3qlYyZBH8G4CzgVuBZ1vHtSAinzhDhceUEX948VqnJ4vIB7FpmytE5MtYXtdvtBt6EVkCvKfodbQtD2zdfFvZWcB3HWbMy78NW9/0KNaOSg1oOLe0zNi2LTOmqrmRRiP421V1exFZBLxJVZ8VkSWaM20y8HVPCTwR+AYWcXkONmp6tKqOuBkmIs8yfK10Vqojp1h7+dhz0X7YyOrVDD9XFM1KqJuvO11GZV6cU9a9fKbcWgxFyt0MM+5Xatt6OxFZjuWXLroBcFyX/ECv2UsjezVIVVeKyFIRmVrFjHn5oFcAS8QSRa6KTKfF4c7nAxeHzmEVM+blT8GmNi3W6ncQGhU6G/i2iBwKrMQ6veuIyPdVtSjKaV38/XkGrURevhs1rR2N9TYUU2ast6MvAPNEpJIZi+DPwCI8Vp067A4tHlHGG6HuURFZu5MZ64L/iqpeJCK7AH+PRcv9EfCmNu65MsOVo7r5rM7FRt8qmbEIfhYWKbBqO3pGVZ8REUTkpap6n4i8rof8I+FcdAlwjYg8jk0pLtIHgpnuaMYi+d1V9Zhwg+4hLILijeTPfPid04x5+Vhthf2P38Hwqb1F58G6+brTZVTmw+/yYRH5e0aascXd8hndCLxFbCT3auya9iFGJntfrqplUZXb5eUHckRvlVQ1bTVsWANeAVyH3R2cDczuIf+2vK2EfxA7eUvF+nv5G7EknZ7v6IfY1JFDgd8DdwJn94pvKzsd2LoDsyg87o9NK3gJdkdqVHls1MnzPbr4UGb3yHbdqHY01tvQWG5HXbSh2zBTfyC2vuMA4IAe8t7PcSM2FfNcLOjE0cBdvSwDTMOiL64DfC18nk1L+LvD4y7AXCzI0a095O8Mj98G9iv63rCcr57vsla+rez9WHTBGeH7nQZM6yH/W2d9Lg7nlmND+7gUuKJXfFvZt4XPsnoJ0zpPfAAzrut2aKNefkl4/Anw7vB3Lh/xm/TyH+2iDRV+h33gD8Zu5L8NWIbdjDy0X3wosxBbF7whZuovwqIh94q/IzweDhyTbYuj3IYmx7ShsbL1vQKDuuE3Y5V4bH3GDsBqzvp4O9Fe/pxQ5gvAZ1qbo/x0OnSkq/LA5uHxjXlbyWsuwTrOF7W++04Xuzp4nJ1oL9/N1uR2NBbb0HhsR9R/0f4W8HFgCpbAeDIlF3KcRiy2TMxnpoMZ64K/HBsBXYYZjpd2aqdN2/CbMS//Q+DfgJnYqNXe2HTiKmU7mrGqPJYigmxbrtiuK5uxSP472PqvO8P56JUU3GDAaca8fBdt6BJgg6bwTdyoaMa64O/EZvPMB/42HFucw7nMmJcf9C1N4+yxROQobOHpLVoth5KLxxZzfw9b2L4YuCWUn6flocuXAXNF5EqGT4UqijDk5R8M2+phK5SIbK42ZWXE/GgReaOOXDfi4rEEpocwfNHvqo9A8RSKM7A7UXcBN4Y56GXrrWrh1RHkJobvUo1oRwPUhmLKjPV2dKWIfByLUJdtE0XnLy8/Mzxmp1QqBUEadGga4f/Don52VNUyInKhqu4bztWa8zpFUwj/KCJnALsBJ4jlqCyb9unl98UiHp6sqk+IyBSGh9EfC7pTRP6Nke2iKLqml18jcLtnjilt0TtFZB1VfUpEJmcOt6atvYy24BReHjOc78VGTJTh080K2zVwmYjch7XRT4rIK7GItkVy8ar6+TAl+Um1pShPA+8vYM8ped+u+S70cuA+Ebmd4W2iaDlMLbyIfFhVfyYFAUXar5d1820SsUjQ+wMHhWO5EaYj+aOwc/XFqrpERDbGUsO017GsfztCXn7QlQK09FgicjIWHr01T7nUjHn5TLnVscAAO2F3RXYEntCCaGNiUQhHSIvzl7l4j0TkX1X1EBEZ8YMmPxCEi++lRGS1iiZ8VPi6FNavqaqu6MA1oh0NchuKKdPPdiQik1T1mbZj62tB9EwReTDnsGpb0IVYvqpijJi3jIhMUdXlwZCPkBasVxORNTEztlhVfx/M2FZFhrwqX2A0svUZMx0kETk757BqcR48F++ox+Wq+t7QTkeYsfZ26uW7rNtkhszYmtgo4aPd8CLyDlX9jQxP05D9AEXmuXESC8IzQqp6w2jyIvIJVT2j6vWybj7nM3wWG5A4IZixo7Q44JWLb5pE5HAsaNvj/a5LL5XMXk2KMGNeft3A7BweX45d6A/s8UcplYh8T1WPEpHLyO/8FN3xqrNOuRehltovRg2/q1bZiMXwIrI9cBYWyl+AJ4CPqerCKu/VKzWtHdXdhmLKxLajCDPm5RcDh6jq/LC/D/BtVc1LLVCbvJ3QGCMWa96qymvGIvgooxE6/p8FpoabJq8FXqeql/eDr1Micoyqnigip5J/LhrVTqvkzEbISkfOYvD+Drz8car6tbrMczcSkbw0B08CC9RSRiQ1RLHXEBUz0gAAIABJREFUfK8Zi+C/gUVavgPrG12lA2CU0jTO+tRaz7Fu2P5EeUSiSryInIklxFyBhRafB5xS1JC9P6iIH+B54fHkks/WXidvR9rFM5QraAPMPP8m7O+KfV/t/FrhsWquw7p5YKQRE5FSI+blg2YBn1LVm8Jr7IKFrW8foWhUOxqANhRTJqodAbeLyAgzRn6etxh+P+AsEZmL5Xtbj5xprnV3QrF1T79heK6wVTht/zNVXR4eKxs0bxmxdBaFHQUdGZXWO2XPxavqe8PjjCr1z+js8B6tZPJ/xNaMFpmx2nivGYswb628YwsK6tpeH68Zc/EMTSWfhN0Qvgv7P28d6rhjG+/6HXh5Vf1aeHTfWPaasQjzNgmbJXVR2N8HWxawjYjsqqpHhde9WVV3yfl95kaLHgW+NBdfTpuulQ9lGnXNz+hV2DWqqhlz8ar6ZRH5CjZ9+0DgNBG5EJilqg8469oYpZG9HivHjM3H8ugUmTEvPwdLQnkP1un8LXBPUeMVkb9T1YVSfTqBi8+U20BV/6vt2OtUdWkO27ojmNuRbnVIYvlMuauxqH3Lw/4U4BxVfVce3zSJJRs+rM2I/VCL8zy5+MCMyE8kOflmmtaOUhuqLhHZCrvIzWXIjB2sqo/0gg9l9sIu3iuAt6rq/TmMa0Sg7hGECCMWVSaUOx5Lt3Ae1uHbH5iiql91VrtrichqwD9gnWKA32EdoMJpwDKUJ3DV+UJE7lLVbUabF5E9VfUyETkg77VU9afd8JlylUa4ZWhqeK4ZU9Udu+Ez5X4FfE1VF4f9LYFjVfUf8/g6JZYi4uMMtaF7gTNV9T86lDuTfDO2HrCsZca64OcDO6ulQ2m19ZuwKLWLtWCmVL+VaZs7A68Hfh72P4iloTh0NPlQZlSu+aHsKwPzWBHTxgtDZmw7oNSMeflQZpvAvxtbQ/hm4BpVPaZKHRsnbUCUmEHasPw0C7Cogh/H8qwUhp338qGMAFsG/pxQ/mrguJIyIyJCYdNkesUvBfbN7H8WO4mUfY6rsQ5Pa38K1unoFX9v2/6E9mOZ53YFfolFOlwC/AJ4e8lr18qHMnmh0O/oBc9QZMnvYQE/3o7d4f0hNlI8JtrRWG5Do9WOQrm9MCP2JypEjfTw2OjwXGAG8C4sQt9hnd6jrg0zVOtm9qcB15XwxwOfwkZM1wE+CXy9w3u4ypAfLbUoTP1q2CjL58K2ByXRlz08Fg59afh//Uv4/d8Qjr265D3mYbNPWpH2NgFu6xcfmEk5x9bvIX838ObM/j7Af5Twv8LWSbb2twR+0UN+SZVj4fjrsBHBX4ftZGCzkteuzGMjicuxwETvx84Vx2HnijcXvUcoOx+Y2NZ2f4sF8Bhxno/glzL8t78usDT8nXd9/PucYweU1L9ufn72t4tFOZ3fLz7DvRJ4ZSfOy2NpR/4bC0r0OPAY8NWK77ENdv66D8sReidwYrc8cCQ2y+AqzAy/JByfADxQ9Tto2tb3CgzihtOMeflMuddgySe/DzyArfErYr2daC8/BYtydhEWOv8M4GUd6l+5Ix3JnxZ+sB8N25XAqTncHtjdwgPDCWFb4GNYJMn39IF3GTEvH8pcX7L9Zqy0o7HahkajHWXKucxYBH8UmRtUWAdrVgnvNWNe/hOhzu/BIqr+B7BnCV/ZiMWWwczM/lgndUL4e14O5zJjEfw5WLCE9uNHAD8tqf9u4XUfA87HosG+vV98KOM1Y15+KyzB80mhTnOA15Twlc1YJH8Blhbh7WH7V+CCHM5lxiL4K/P+N9g158oO/zOvGfPyB2HnyLNDW1+G5ZZbCzgph78R6/ivhU33u4xyw103v5RMygAsJ97SPvLH4jBjHh5Lq3QNMCNzbGPsent0yXu4zFgEfxwF+TeBLcrad5O3vldgkDccZqwqj12U/x34z8Cch91V3oaSfGb4O9Ex5u0w4JFQt50qfD+VOtKxfCjzAawT9C/ABwqYucA2Oce3Bm7oA+8yYl6+yzbdqHY0VtvQaLSjzPNeM+biA7MGJSO2bazXjLn4UGYX4HmsE/s3HdhKRqybMlhOx0uxjtBjWP6t6TncOTjMWAR/X0kdCzt94fn1sBsO76VkRGwUea8Zc/GhjGeEu5IZ64KfBByNJWO/OPydN1rpMmMRfJlB7tSGvGbMxYcyUzDT+n5KRqsDK8A/A78P28w+8wcCD4fP+tPw2Q/oB4/TjEXwd5LzO8dGBctyhLrMWAR/UM6x75T938bC1vcKDNqG04xF8KdgdySnRNTNa8Y8ne5rgXOxqKBbAbdhOZw6vUfHjnSX/DTCVApgTWDtHKasAzTiubr5UW6vLw9t8BTgB61tLLWjsdiGRrsd4TBjXh6bQrgUeDDsbwvM7lCmshnz8sA/YYZwJhZY5g5yTHKGn04FI9ZtmYrfpcuMRfAdE7O3HXtj2TbafE75xkxPpqIZi+VDmY6/S5xmLIJfWMIXLjPIMJXNWCS/IbYu+62trYSdjK3fmoPFQfg85ctuauVDmb/JfN4q58ZaeJxmLIK/p+S9y55zmbEI/gpg/8z+6XS42TkWthSNs/eajo1iHK0hqEMveVXNDbveSSJyLXaB2xLYCJglIjeq6j/3ggdOU9VLwt9PiMhODE9qXKQ7gBWqeq2IrCkia2t5yoDKvIgcgk2NnYytAdkQ+DHwzjb06ZL3y3uubh4AEXk58BGsjaz6rWpxfhsXH3QFNo9/MfBiCdd6jya2o7HYhmLKxLajPbE1OKsDM0RkW2x9WVFoaxePTd3ZAesYo6qLxHIrFdXnn4CvYG11a+AKETlQVe/qBY/dDNtFLdDPBSJyMXYn+w15sKo+REEy6CJ5y4jIZth0rlep6pYisjXwPlX9Rhv6/0pe5q85x7z8upIf3VSwtYft+m7OsZaUkVFX6+ZXSURmYb/JrbFIsZeLyKmqenoveOyceLBaj+9BEXkTdlMsv7Kqz4jIj4ErNCcwWbe8iLwPG5Xs9LssO/flnSe8/EaSH91RsPNjJ03AbpCsBmwqIpuq6o294EXkBGx21BKGrmeKzSrJ03ys43+WiKwBnIDlOt6pH3wIJPL3wMaq+nURmSoiO6jqbX3gX6I56XZU9TEReUkP+Ofy6ljhuX1E5BlVPR9ARE7Hbpz0jAdmi8iLWHCWJ1T1oBJ+TChF4xwnEpG9Mp3oVpSqL6jq8b3gA/MqYPuwe5u2RVXM4Vd1pFV1E7HcSj9W1faOdCy/COuE3qpDEd4Wq+pWbdwT5F8MBOs0vmI0+Uy5eeQYMS2OHufiQ5kRkTfL1LR2NFbbUEyZLtrRQqzTPDfzGe5R1S17xM9X1TfL8EiKd2tx1NhLgI+3/q8isgNwhrZFhY3lC15jdVXN7UA4jFh0GRG5AQueckbZdyoiy7BpXyNeAgsmsEmX/NlFnwniwun3SyJyFPD9YMYQyz17SlHHzMsHZnWGUo4sVdXnS9hVZkxVq9xU8fJ5v8u8c9F/YbOFRrwEtn76VV3yB+TVr6UO15tcM1bymb38UmBrVX22rI4Zfqqq/mfbsbeWmMm6+R9hn/MdqrqFiLwCuFpVtx9tvqxvkPdcBL+S/JsJgo1w5xlEgmmejUWMbpmxI/NYDy/Dc5Wujc3WuAX4KozMWTrWlMzeOFKEGfN0uvfFLlxzsR/rW4DPqeovSspU6kh3wd+qqm9qdUKD0bijvRMqBaGCW9KRIYZr5TPlvEbMxYcyRwN/wfJZrbpAlp3YmtSOxmobiinTRTvymjEvPwu4DpuitA82LfglmhPOu0hlZszLi8hrgFOxqZ+KhV4/UotTTVQyYt2UEZHbVXX7tu90kapu28a5zFjd5k1qzmfp5XPqV9mMeXkReTu2rukh7Fy0Eba+qaijXsmMdcFX+l16zVg35s2rCDPm5a8EPqiqf3HU6X3YdE+wtc+X9YtvXcOlesqS2nivGYs1b1XlNWMR/IMM5SptPbakqlo4W2UsKE3jHCfK6USfKiJlnWgXD3wJ2F6H7r6/Elt/VWj2gGdV9TkRab3napTksIrgbxCRLwJriMhuWKj0ESfaok5ykermMzpPbCSqqhHz8mDTJU7C/n+t71IZmbwZaGQ7GpNtKKZMF+1oiYjsB0wUG8k8Agsw0iv+cOz/9iwWdOIqLDVBrorMGLaus2seC+bwb1jkNYAPh2O7FfBrquptrTYRVJhzLrLMf4vIJoS2JiL/iK0/HCavOevWzFVQK9F2bn5KRiborptfpTwzJiJlZszFY1NMd9cwxVJsNPcC4O8K+OdV9cm2NlF2bvHylX6XXnPWSzNXQcuwcP+VzFsE/1dgkYhcx/BrYNHSh+9gNyLPD4eOEJEdVfWL/eCB50VkIkPniVdSvryiNl5VJ5a8Ttd8hBYy0oztEba8PouLV9UZNda9/9IGLBxMW/0blrh1g8z+KykPFe7lF7ftT2g/llPmROCL2ML33bBF6t/sIT8Bi953EWYWDoHyxdFN2rDAJk9gnZMHw7asV3wos4wKke+a2o5SG6r0P1sT+CYWiXBB+LsscISLj6jPNVhUuNXC9lEsWW2v+EVVjmWeuxJby9XK8faPdA4h7yqDdSyuxTqjfwRupgcBXUaxDdWdz9LFB2YhmWAl2IhdWfAQL393lWOZ52YB+2EpHl6L3aD4cQ/5Wn+Xo9SOfgncj0Vl7hgQLII/IG8r+x+TCYSHRdct+x/Xze+PTTl8JPx/l2IjlX3hx+uG3XjaD1sn/hHgI/2uU7dbmsY5TtQ+PUREJmCd7qIpI17+JGzh+wXh0Iewk9r/LanTBCy08u7YXZergJ9oQaP08mNdYutxdtCcRc+94EOZq4G9VDUvmEMe36h2lNpQ/yQil1EyEqHF62rypi+OONYFfx02ktdqQzOBA7V4XebGwJnYBf5x7CbJh9WCsOQqpkwotxbW+SsLitE4ici9qrpFZn8ClhNui37wgcmbwlg23djLn4WNevwsHNofS/D9sQJ+TWyEO3tuOV5Vn+kFPwgqmjKqxevQXXxEfe7G0k78b9ifjE2rLWxDdfKB2RwLACZYPtF7O3yGWvkmSixw23SGB6I7txe8iJyH3chbBKwcwksD3TVeyeyNE0V0omPM297YVCuAm1T14l7UPVYi8l5sOtk07Ecu2I82L+pcttw6gavUIauLjzBiLj6UuRj4WywnX5VpL+OqHXnbkIhMau+sicj6ZQbcW6Yq7zVjEXxrDeHeWDjvVqd4JvBnVT26oP5eM+blp2GjJDuGzzMPOELbAiXklHMbsaplJC5Sbm0KRuOzwFRVPSRMC3ydql5ewJ+GjT5lf8f3q+rh/eBDGa8Z8/IvxWZLrDoXAT/UiuvHei0R2Q6blTCd4W2o0DjUXJ+8iJxPAgtU9dJRrsuFqrqviCwm5xxWYsZmAt/Brn+Cra37vKr+vB98KPMKbH1o9n98R7/4uiUihwM/U9XHK/IuMxbB3wu8ftBuACezN47k7URX5cMc8GtVdVdnfbwdaS9/P9YRXVzlhysi22MRm9YOr/0E8DFVXdgn3mvEXHwo475z2qR21MA2tBg4RFXnh/19gG+r6ma9KlOV95qxLszbAlXdrtOxzHMuM+bhQxs6QlX/Je+1Cl7fbcS8ZcQfWddrxrz8z7FpjR9Riya6JpYUPne0NJTZGwuYBHBjxd99nbzLjHn40I7OVdX9y+rQVsZlxiL4pVhQoPY29HAB7zJjEfyZwObYFHew4EwPAuthyweOyrAuMxbBT1HV5eFcMUJ535GICPAabK1tNoDYo3mvUTcfyhyPTVN/gMwaelXNTUFSNx/KeM2Yl/8G8H+wlEhnYdO3C6+1XjMWwV+EXUOqpE4bO9IGzCVNW70bNk/8+rr4UOY6YF1nmfuxUZ9Ka6Ai+OvJSUxfwt8NvCWzvwud5+PXyXvXH7j4QWhHDWxDW2Frak7CFuXPAV7TyzIR/IIqx7rg78XyNrX2ZwD3lrSJo51tqDIfytzm5Odh+dMOrPq78ZahQrLpNv7nwDGE5MLYeq2ydYdefkF4vDNzrHAtbdO20C7Or4sPZW7G0iJU5ZcC7wvtf1pr6yF/s7P+Z2KpWg4P21xshHw28L0e8POxkdHW/mrAb8N3/bs2dkp4nJa35by2iy/5DiaQSZCd83xpXIE+8Esj2lxtfCjzDey6eSGWuqBTUngXH8oI8C4sBcj9wLeATQrYi8is763w2l7+emxq/lWh7c8GZnu+syZuKRrnOJCqrhSRF0VkXVV9std80F+AxSJyDZnwu1o+TekPWOek6vCylz8GS8B8A8NHuooS465U1Zsy3M0iUhZhr1ZeRyGqmgyFG25/rRHROBvajhrVhlR1sYh8EzgPS1b8Vi0I+R9bJuI91hKRjVV1GYCIzADW6iF/NDBXbM2oYB2yTxTUfWWY2lRp5M3LB90SpgX+nOFtqGiq0iRV/Yzj9WPKeCPlbqKqHwqfHVX9axgpKJKXf04s/5T1tCxSaN4I1wpyzg+Z+g8bEa+bzxxfKSLTpGLKDi8ftAxrS7MZ3o6Krh+Pqersiq8dw39NRH6C3RDLtqGiiKVbAzur6koAsRxrN2E3GRf3gH8F8DJs9A/sHDE5fNfD2pKGURJtG2ETW5s5E3i4G15sacRhWFL32VhQp09jo913MRQNs113iMj2qnp7wfOjzd8DvBwoTWc0ijyq+mUR+Qq2tvRA4DQRuRCYpaoPdMuHMioijwKPYiOhrwB+ISLXqOoxbfj6wO9E5DaG/w5y14hH8McWHB/TSmZv/Mjbifbyv6IkTHaBvGbMy38T+xyTgNWLKiEirdx0N4jIGdi6EcXWjcwdbT5TrrIRi+GDslPtJmHh6icXsNC8dtSINtSSWM65TbCO02bA5SJyqqqe3qsyEe9R2YzF8Ko6J0wb3Dwcuk/L1zV5zZiXb01F/Hq2mlhOszzFpCzxlnGlOKGiGeuC/xo2IryRiJwP7IxN7xomVV07vN7xWKqI87A2sT8WMXNU+TZ5zZiXfyBsE7Cp953kNWNe/kDsN/YSMgnGKT5fVjZjkfyJWJqDuQytR/uW2DrWa7Og14xFmLfzsNGY3wIHY9NjBVvDviin7i29CfiwiDyEtYnWtP6idZB1898G7hSRe6hmTOrmW897zJiLF5EjsSnx/w38BMur+3ww9r/HrtlZHVtW1xy5eI1PcdRopTV740RSYxQsEdkW2BSLnlY5spNYQJG/MHINwnE94ksTI2e460ueVm2bz143nym3XmZ3lRFT1a/2gi+SiCxU1dxcUk1rR01pQxn+KOD7rZFDEVkXOEVVD+pVmcj3eCnVzVgM74l2lvd7KPsduHivROQwzNQ/wfB1LIU3SbxlxB9Zdzfgy8DrsbQEOwMfVdW5veBDmfWAN2Md0PlldZOcxMt5x0aLD89/Le94yW+5Mi+Wf2waFiTmiaI6tJX5GfabWULGjGlxABgvv1RVX1elLoE/CGsTc8mYMexm47Gq+rlu+FBmCrBD2L1dVf9UUJdLGTJj78TyKgpwZJ4Zi+BXRX0WW2+5HFu/WhrZVBxr/EaJX4KlmWi/PuUakLr5UKbdjF2SNWOqukmX/HHAWXnfiYhs4elTdiMRuVlVd5GRsw0qBfZrupLZGwfydqI9vIh8FUtavBC7i/VtVf3XivXydqS9/IlYwI+rq5ZpusqMWAyfGXUEu4O9HfDJvE5WE9tRE9tQGGGZqiEZcx1lIviBD1UtIm/C1h1tgnVmPlaxncakLPGat5hIuZXNWFW+7fc+QlowWioWYOZ0bE2NYlPpDlPVnfrEu8yYhxeRgzGT8wAwA/i4VphuGWHGvPzZwEmq+jtHmUpmrAt+Q4aCXQGgOUnqvWYsgr9DVd9YtJ/DTwIOxa5Pi7EphoVLKurmM+VuV9XtO3GjxYcyLjMWwR+kqrPajn1HVT/fdsxlxgbdvHmVzN6Ay9uJjuCXANurrRFZD5hT9WTi7UhH8CuwqSjPAs/T4Ucu/gh7dfOVjVgMH8pkR01ewBKyn9xuIprajhrYhvYETsYWwc8IhvfrWjJNxlsmgm9EqGqvGYvgFwBfwIJMvA84WFXfVVanUC7GiHnTolSKlOs1YxF87CyD6cD3sRFDBW4BjtKCvIJ18l4zFsHfA+yqqo+J5VM8X1V3LOIz5VxmLIK/F/stPIi1oU5TAiubsRheRE7AliK0j0yOOA9FmDEvv5KhqbkCrAH8lWIj8HPsfH4T8A/Aw6p6ZMnr18pnyp2C/W9nM/w8UXQTplY+lKlkxrrgr8B+Y+eH/dOxNdGFM1XqUJtBvxszrB0N+lhRMnsDLm8nOoJvPylXHnmK6EhX5kVEgI20Q26ttjLe8Oh185WMWCzvUVPbUQPb0EJsbdhcVX1DOFY6mugtE8E3IlS114xF8K4OYoaLSVniTYtSaTqz14zFmrexLK8Zi+Bj25HLjEXw3imBlc1YJL8U2For5B2MMGMu3isZPnK4GhbBt8xM1spnytU6xd3LhzIuMxbBr4GZz7Ow6J1P5BljrxmL4KMM+lhRCtAy+HpWw91nVf0fsXnTveQ3FlvwDnYi3iSzX7jwN3Sk/7ZqR9rLq6qKyK+xMPVV5Y2wVyuvzryFXh5AbG3WPowcbfx6G9q4dtTQNvS8qj4pwwMhvlgER5bx8vdgefOq5g3y8lWjnU1Q1WvC3xeJyBc6vK6Xf7lYrrbcfS0OfHFJ2DyqXEZsGtruWiFnW92/+bbvJ+/1hn1HInKMqp4oIqdCbvCn9pHJWvmg51T1sfD8snAOK5OXf40Mzzk3bL/kJsC7O7xuNB/a0FWqunlHeEh7YbkWqyaB9/LLsGAxHXlVnVjxNaP4CD2fea8XpDRw7ajwhGvkj1T1wo7wKPAZ7QPMFpEXGTJjZaNulXgRyQaCOxg7n94CHCcik3VksKufMmTG3oPdbCszY17+9RmDPgu4rYQdc0pmb/Dl7UR7+fe37Z9cpVLejnRkx9sb9tgbYa9W3mHEovigS7Hoawspv2g3rh01tA0tEZH9gIliESqPwHKy9bKMl29KqGqvGfPyNwB7FuznRi30GLHYMuoI+x9hxlw8Q9/HBsBOwG/C/q5YG2rnW9NmF5S9zyjy4DdjXr49EMnCThXymjEvH9rQUhGZWvVmFQ4zFsn/FYvG2R5NdFTX6kZqGxF5KvwtwBphv2jksG4eVX1RRI7B8tN1VN2814xFmLeF2HlZMo97hC0vUrHXjHl5t0EfS0rTOAdcIvK2sue1LQqTl+9GIvJT4LSqHekI/j5sCP9hKoQ9Fn+Evbr5OQwZsdbaKVT1u73gQ5mqEUsb2Y4a2IbWxMLr7x7Yq4DjtSQqnLdMBJ/7vyj6H3j5qhJbo1Qk1bYohF6+i3rdDLyjkxHrpoyInAtsgU1XKgz7n/nMuWZMVd/bDZ8pdzWWBH552J8CnKMV1jj2W1IwJbYlHTk11sV3Ua9LgcOrmrEI/kbgDVinNduGimbP/BLYhpGpHYqmGnt5V2TmpM4Ske9gUSzb08wU3RCujZehVE5ZM5YpMrzf4uW9koat+xxrSmYvqW+K6Eh7ee8aB2+Evbr5WiNNhjJnAqeqal7S3MaraW1oPEmKE2KPqYtjVSPWTRnxpwlwmbEI/l5V3SKzPwGLmrtFG1cahbLdaNTNN1kRZszLe2/a1JYmJ6keBcPUrrIbwrXyoyWpEP3Za8YG3bx5laZxJvVT3rvILl5VHxaRXYDXqurZYuG3X1ZS5H7sZFBVdfPzRGQrhxGrzIsFLXgROwccGIxopQhvDVMj2pCIXEa+8Wm9Tt6aQ1eZCN5lxry8hoTYAyBv8mx3mZapE5E1tVoEz410eMCbPwNTe8hfJyJXYTnUwAJzXJvD7Qj8IXC3MvxufZ7q5pusr9TJq+oN4ebTa1X12jDCX7i2zWvSqvIicqGq7isii8lfZzlWrh2Nk6rOaBLfUhUzFstLQfRnYBivzVv3OaaURvaSeioRWQfrGK6oyI/oSKtq3t0nNx/upm+HLTrfTEReDVykqjsX8N4Ie7XwbUbstdhaikIj5uVDmceBbfPqGerUl5ErEZmkbdMRRWT9stHQJrShzF33vbHgJj8L+zOBP6vq0Tmv7SoT8x5J1eUwYu4yIrIjMAtra1NFZBvgE6r6qQL+NOy3nDVj96vq4b3gQ5m9gbeE3RtV9eIcZiKwG9bGtgZ+DVygqksKXrNWvunKM2Nl10IPL7be++PAZFXdRGy97o9V9Z1tnMuMRfBTVHV5mvXQe4U28Bksp+DHw//4dap6eT/4UKYRqXuSulMye+NMEWasEi8i22Ohc9fGTMYTWF6swsXtEWbMyy/CpsncoUMh6u8uuvNY97SXqrzXiMUYN6kYTjynnMuMRfCLgUNUdX7Y3wfL0bdZAd+0NrRAVbfrdKybMjHvMZ4UOjSfxTo0h1ToALmMWEwZEbkV+EdgtlZPydHRjHXDeyUWAGomcBJwnKqe1k++bsnwoC4tPQksUNVLc/hKZqwLfhGW8PzWTBtaFeI/w7nMWK/Mm9hU4JkaQu6PBYnI4cDPVPXxhvA/x9bcf0RVtwznsnmqmnt9r5sPZRqRuiepO6VpnONE7WZMRErNmJfHOj6fUtWbQvldgLOxu7VF+gChIw2gqn8SkbIpUV7+OVVVEdFQp7VK2NqmvUTwDzrvjnp5gA1EpDANhBavV7pdREaYMSDXjEXw+wFnichc4NXAelhOuSI1qg0Ba4nIxqq6LPAzsLx+vSwT8x6NUYQZc/HYeWchNkUQ4I/ARVgU3Dx9D5veOxtAVe8Skbd2+BjuMqr6Bxke4W1lERv4X5ETQbQbXoqn6rZeIy/f5EuxCHkzsalZPwAKjWTdfKac14y5eGASsDnWdsAiHT+IRVrcVVWPauMPI5gxAFX9vYhsUPIRvPyzqvpcqw2J5W/LG4lbHh7bTd0E7Dt+uBs+3AA+DNgQa//XAJ/GfqN3AWPG7AGvwq5Rd2D9nas6mJS6+U1U9UO62TwqAAAcjElEQVQiMhNALU9t2dTmunloTuqepC6UzN74kdeMefmVLRZAVW8WkcIElkHejrSXv1BEzsBCth8CfAz4SREsQ9GkhknLFzvXwXuNWIxxm4itPfOukfGaMRevqotF5JvAecAK4K2q+kjJ6zeqDQFHA3PF1kAKMA34RIf38JaJeY8myWvGvLy7Q+M1YhFl/iC2jkVF5CVYvqd72yGvGfPyGtZZisjxWGfsPKwN7Q9MyanPucCWwBXYaNs9Re81GnybvGbMy28N7KyqK0Ndf4Tl7NoFyFsXXcmMdcHfICJfxML47wZ8CrisHfKasQjzdh7wOPBbLMT+F7E2tJeqLiqpf+Okql8Wka9gkY0PBE4TkQuBWar6wGjzwHNiScZb16dNKE+FUTcPzUndk9SFktkbP/KasUq8iLSmAt4QOsUXYCeSDwFzO9TJ25F28ap6crgoPgW8DviqDiVpzlN2Gtwk4IPA5AK2Tt5rxGKM23Itz7+XK68Z8/Ji+XA2wTpamwGXi8ipqnp6QZFGtSFVnRNGnlr5s+7TDkmKvWVi3qNhqvtutLdDU8mIdVnmUOD7WIf6j8DVWGd9mLxmzMtn9D5V3Saz/yMRuQv4ahv3YSyi3ZHAEZmvvSiiXd18Vl4z5uVfgZ1Xnwz7a2FTLleKSF57qmTGuuA/DxwU6voJ4ApV/dcczmvGvPzGOpS/7CdYu5uqJellmqxw8+9R4FHgBez//gsRuUZVjxll/lhgDrCRiJwP7IyZxCLVzbfKeOTitYfpvJJKpKppG+ANeGPYvgecAbwdeBvwQ+CUHvDXl2y/qVC/3bA1GicDu/WSB06ocqzDaywcbR5bH+Z5DRcfytwZ2Z5mYSZ+BjaN7T7gsB7yRxHWEof9dbG7oGOmDWH5zvYDPtLaKtTJVSbmPZqyYcm712i1W8zc39ZDfjcsofpj2KjEQ8DbS/j1A/dn4L+wwDeTO3wGVxnMZHQ8lnnurirHuuDnYYZwIhZRdH9s7U7f24ejHS0F1s3srwssDX+POL9F8AdhI39nA+dgga8OxkzfSTn8BOAQbOTwF9ja47L6e/kjKx5bnPl7Ymifk0pe18vfUbY/ljbsJsNCLFfpB4GXZP43D4w2H55bD5vW/F5g/QqfoVa+xu/+5vC4AruZ2tpWAE/1u20M2tb3CqSt5n+w04x5+S7r5upIR/AjLkLA3SX8GzPbdtjd+LIOUy08TiPm5UOZ0s5sSTmXGfPygVkDW5M1FtvQeVhH+ofAqWH7QYfP4CoT8x5N2vCbMRcfylTu0OA0YjFlCtpRYScZpxmL4KcDl2IJlh8DLgGm97ttONuR14y5+FBmCvD+sL26Q30qmbEu+Lw2lGdSXWYsgl/J8I75C4zRTjpwHDCt4Lkt+sBfV+XYaPA4zZiXT9vobikaZ1JPJCIvx0YYpjM8t0puuN1QZkRESCmPdFiJF5FPYlNiNsZyYbW0NnCLqn644PWvz+y+gHUqT1bVpaPJi8hkVf3fvNcoeF0X363CFLmpRZ+zG15E9sRG3FZX1Rkisi3wdS1ONNy0NuQOI+0tE/MeTZOIrAe8GZsuNl9LUmtU5TNTynOlqncUvHZemyiNVlu1jFjUzp2wmx7/knlqHeADOnwqZbbcdGza587YdNRbgKNU9aFe8IMiseTxO4Td21X1Tz3mN8TWxGavaTcWsHn//zs1RM6M5cP05f2w6aY3ZZ5aG3hRR6ZeSMmnK0pEDlLVWW3HvqOqnx9NXkQmAWtiN9XfztCSjHWAOaq6+Wjyo6FQp0OBTYG7gbNUtVOch6RIpTV740ReMxZh3q4A5mPrCV7sUJdVHWkRuTvz1NpYJ6UrHvg34Eos6mP2pLqizBSp6q5l9R4t3mvcRtnorTJjQBUz5uKx+f47ENZ7quoiERkR8KapbQh/JLKYMjHv0XflmLFW/aeKyNR2M+blge+WvL3SFhgoY8ReKcMDHK1DQbLqiDKrY+u+VmN48vWnsFQM+ZU1k/b+oudjeRE5RlVPFJFTyQ8WVXhzrqGagI1MrgZsKiKbFpkxLy8iJ2Brz5cwdE1T4MY2rmXGZojI7MxTawMjzhVeHhu1XY5NHc628RVYJ3mYNCWf9mgfEXlGQ7oIETkdW08/2vwnsBtCr8amfbbM2FNAXgqSunm3GYswbz8FnsduYLwHy0F8ZAmf1IXSyN44kYjMI8eMaXFOOC9fOW+biKyLLVKu1JH28m1lJ2Lhj7OG9T8L2JdiEdqmt/G5gUzq5psoEVmIdZrnaoV8YRH8fFV9c/YOd8HIW1Pb0PVYzsPKYaS9ZWLeowlqG9lul6pquxlz8RH1eRt2l/tQ4MeZp1YAl6nq73tRJpSbphXSo3jNWAS/p6peJs4coU1UkRkr+d14+aXA1toh+JFYfroZ5JxbsCnfL3TDJ9WnMOtkNpYW4d3AE6paaDhGgT9cVU911L82XiwnX8uM/QPwcIe6e/lVOSLFItHeVrUPmeRXMnvjRB4zFskfDfwFC4ee7YD2rCPt5UXk09hI0Z8ZfnEvmuI3B4u8tpBMGHVVzR0xqJtvoqqasS74WcB1WCdoH+AIbFH7oSV1alIbelvecS2JOOYtE/Me40FiScULpZaHLq9cJSPWTRkR2Qz4Z0be6Gk3uC4zNkjmzauqZqwL/krgg6r6ly6q2TOF9n0CsAE2MjPw0yzrkohko2Cvja1ZvYUQjba931I331Z2J0aeJ84dbd5rxiL4YX1Mb58zyadk9saJvGYsgj8M+CbwBEN3mFULcs6FMt6OtJe/H3iTqv5PUR3a+MIRp37wTZTXjEXwawJfwvISCRbF7HgtCOvdtDaUVCyvGYvgzw5/boBNtfxN2N8VC1by3oJ6VTJi3ZQRS2vwY0be6FlY9B51qG3a4Ag1fXQ4K68Zi+B/CWyDnb+y18CipQ8uMxbB3w/sqaqd0oIkdZAM5byVzGNLI/otdfOZcudh0YYXMXSe0JI2VxvvNWMR/LhdK9oPJbM3TuQ1YxH8MmAH7RBooa2M14x5+eux0PqVpsWIyJnAqaqal3Np1PkmKsKMufiI+jSiDUlxcuvCC5e3TMx7NEleM9aFebsaOEBVl4f9KcA5qvquAt5txLxlRGShqv5d0etlOJcZi+AfA/6A5UO9leGd0DE1Ohxhxry8a7TUa8Yi+FtUdecqbNLYlNQcrMvDe81YMm/NVgrQMn70WWBThxnz8vdjP2yP/sBQwto6+GXAXBH5NcMv7qdkIRG5BxvlWQ04MBjXZxk6SbWvF6uVb7JU9a+YeftSL3kRuYx8I9N6naIRh0a0IQ3JrT3ylol5jyZJVQ+EVWbs9e1mrFs+o41abNCfgakl/Auq+qPqnySqzGUi8ingYspnSuxIiRnLkZf/GyyVRStIyK+BC1R1SbWP0SjNDlstfJGpK9GfnaNuXn6B2LqoSxjehnKnJydVU1OmTQbVHayrMq8p0M9AKZm98SOvGfPyTwOLwkhIx7umQZU60l3w/xm21cNWpA2xgBdVVTffOHnNWIR5Ozk87o1djH4W9mdinfUiNaUNJVWX14x5+etE5CrMAIEF5bi2hK9qxLop0xol+lzmmGKpPbLymjEXr6orgTnAHLGAUTOx38Nxqpobla+p8pqxqryIXKiq+4rIYvKD3hTdnPOaMS+/DnZN3j1bHSCZvUgVTWsEcs1Y3TwWcfV3IlI1+FbdfNKAKJm98SOvGfPyl4TNI29H2sWr6nEV6/Gg+gI01M03UV4z5uJb08dE5Luqul3mqctEZEFJvZrShpKqy2vGXLyqfjqsh3pLOHSmql5c8vpVjVh0GVWdUfJaWc5lxmLMW+D2COx04AeYaR0T8pqxCPPWiiCYO024RF4z5uJbI91JPdV2+HKW1s0fW5EbLT5pQJTW7I0TRaw/GLPR3SJGoR4BikZ28qZ91so3WSKyoM2M5R7rgr8X2ENVl4X9GcAVqrpFbz5BNXUxrTSpgtrM2I0dzJibb4okIkJojhmbjeWs+mPBe1TmReRcYEssL+q/q+o9jo/TCInIFFVdLpbCYITab6x5+ZL3nQDM1JAzbbQkBWk1WuoweyapRCJyEXBE28yBvvFJSXUpjeyNE9U15aUlGYo+1f46eQmxR2tKYFVNxBIfd1rvMlp8k7WWiGzcZsbW6iF/NDYysQz7vqZhCWGHqYFtKMmhYHIqTz+rwpcEsGm9RntAgRgj5i2zZxlO22dqM2PHdTJjXh74MDZr40jgCJFVp6QxE0Sh1XHOMXUTMMP7cDe8iKwDHIZNv58NXAN8GlvHfhdwfhvvMmMR5q1sZkNSd2rEtMmSc1dPgnV5+aTBUxrZGyfymLFIfr3M7iTgg8BkVf1qDpubJyzzHsMiwnl5r6T+HIQDkz9GRN4NnImtfVtlxlT1ql7wocxLgc3D7n2akxeraW0oqbMizJiLz5Q7HgtAcB7W5vYHprSfi2Qo2mfBy+vHcl7bXcYjEXmRoYh22c9e1Ilz8YOgTmZMVd/fJX8p8DjwW+CdDKVGOFJVF+XUJ3cWTEvtN069fFJ9KrouFF0P6uaTkupSMnvjRB4zFsMXvEalcOP9lmQSfjeBb7qqmLEueVe0s6SxpapmrAv+LlXdptOxpLGpCDPm5bPJoSdibW+q9ihdTFJSUtJoK5m9cSyvGSvjRSQ7cjUBW5j8ybHQwRKRyVoeeW9U+abLa8Y8fFH0srQuZXDkNWMR/DzgdODfsdGumcBhqrpTTz5AUl/lNWMRvCs5dNLYk4jcrKq75MweKBpBr5VPSqpbac3eOFGBGSv8/3t54LuZv18AHgL2dVe0BoVpPKqqK/Ke9xqxuvkmq8iM0bvQ097oZbVKRCa1dwpFZH2tnn8yaaSeFpH9GW7Gnu4hvx/w/bApcEs4ljQYer71h6quFJFHOoy6efltROSp8LcAa4T91FEfEKnqLuGxUu7SuvmkpLqVRvbGicRSKLTUMmMnq+rSXvBd1q3UjMXyIrI9cBawNnahfgL4mKou7LLK41Zi0TIrm7EI3hvtzGXGIvjFwCGqOj/s7wN8W1U3q1K/pJESkemYEduZITN2lKo+1Au+iRKRNbE1YlNV9RAReS3wOlW9vM9VG3MSkZUMmX0B1sBSGBSNsrj4pkpEfpBz+ElggapeOtr1GcsSkUnAocCmwN1Y9NoX+sVnyh0O/ExVH6/4OWrlkwZHaWRvnEhVd62TD+uy9mHkdL2vl5QZZsZEpNSMeXlgFvApVb0plN8FOBsoSoqb1Fn3YHnzqoaS9vLeaGe3i8gIMwYUmTEvvx9wlojMBV4NrAe8o+JnScpRMGnv78R5eRE5RlVPlIJoh0VTgWOMWESZs4GFwI5h/4/ARUAye06p6sQ6+Vh5zViEeZuErX2+KOzvAzyIjUTuqqpHxdV8XOqn2IjvTcB7gL9lKL9iP/iWXoVdo+7A+jpXdbhRWjefNCBKI3vjRF4zFsHPwS5UCxmaroeqfjePD2XuxtbSZM3YD3VkkttYfkRglLT+ojuFEd9tgUpmLIL3RjvbCrtozWXIjB2sqo/0gg9l9sKCg6wA3qqq9xexScXymrEIfk9VvUz8OUV/jp23PqKqWwYjN09Vty35LK4yEnJLZs9JKWjMYElEziTfjK0HLGs3YxH8fGBnVV0Z9lfDzMQuwGJVfX0dn2sQ1baOczXgtrJ+Qd18W1kBdgcOxJY1XAjMUtUH+sEnDYbSyN740aUMmbHSaIiR/GtU9d3OOq1sGTcAVb1ZRMqmOlTiM+sNbxCRM4ALsM7ih7BOflK8jq2TLzJ1JfxiEfkmw81YoXHz8iIyC1tzuDU2+ne5iJyqqqd76pkEwL3hsWreMBevqpeFR2/o+k1U9UMiMjOU/2voEPWyzHMisgbBtIrIJlQ7ryaNHW3NcDP2IzJmrAf8K7B8rU+G/bWwCNkrRSS1JZ+y6zhf6Pxzr51fJVVVEXkUeBRbQvMK4Bcico2qHjPafNJgKJm98SOvGfPy80RkK1XNu0gNk9eMRZi39tHEr2X+TkPZXSjCjFXiJTLpq9eMRZi3xdjInwIPisibgFOqfKak4fKaMS8vIrM7vF7RVOAYI+Yt8zVgDrCRiJyPrT/8aIf3SBpb8poxL38isChMKRfgrcC3RGQt4NrefYxxIW8Qnrp5A0WOBD4C/DfwE+Bzqvq8iEwAfg8cM5p80uAomb3xo8pmzMOLyD3Ai1hbOlBElmGdntZJLW+KpdeMuXh1rjdM6iyvGfPyGh+1zGvGXLyqfk9E1hCRqaq6VFWfBA6KrOu4lteMRZi3HYE/YDeDbsXaWhXFGDFXGVW9JqyTeXOo15GaIroOmrxmzMWr6iwRuQLYIRz6oqr+Kfz9uV5+kEGX1rzu08tnNBnYW1Ufbnu9F0XkvX3gkwZEac3egKvNjL0WKDVjEfzj2JqsXLWfVEZTIvJy7C7WdIavO0w52wZIYYRlqlaMFOvhRWRP4GRgdVWdISLbAl8vGSVKKpCIPEaJGWsfBY7gJwK7YakZtgZ+DVygqksq1G09hozY/CpGrEoZGZ7CZoRU9Y5O75M0diQiUxgyY7dnzFiv+A2BaQy/nt0YX+OkJklEDlLVWW3HvqOqn+8HnzQ4SiN7g68NKTFjPeAfjDV0XjMWYd6uAOZjozkvxtQxqdnKmjGgoxnz8tiawx0I04VVdZGIbNzTDzF+9DcMmbH96GzGXHxY+zQHmCMWYGomMFdEjlPV09r5HCPWihg7NYzkjjBiEWUKA1RhI98psutgaQLwGHZ92lRENu1gxirzInICtnRhCUPXMwWS2Rsc7SMiz6jq+QAicjoWhbVffNKAKJm9wZfXjHn5DUTkM0VPqmrZlDqvGfPyk1S1sG5JA6Fj8ZkxL/+8qj7ZtsA+3TiIkNeMeXlYFUV4j8BOB34AXFxQpRgj5iqTppSPH3nNWIR52wtL75GCsQyu9gFmi8iLwLuBJ1S1bNlA3XzSgCiZvcGX14x5+YnYIvPq4aaG5DVjXv48ETkEy2WVDfv/v47XSGq2vGbMyy8Rkf2AiWK51I4A5kXXdpzLacZcvIicC2yJ3RQ6TlXvKatLjBHzlhGRvTu83q+8dUhqrLxmzMsvA15CiuI6cBKRyZndg4FLgFuA40RkcnufpW4+afCUzN7gy2vGvPxyLUmc3kFeM+blnwNOAr7EULAQBdI0vMGR14x5+cOx9vMstnbsKuD4ntR8nMlrxrw88GHgaSx58REZQ18URMhtxCLK7BkeNwB2An4T9nfF2l0ye4Mjrxnz8n/FArpcx/DrX1qDPva1EOubSOZxj7Dl9Vnq5pMGTClAy4BLnEnEI/gRicsdZQ8Dvgk8QcaMqWruiSeCXwbskKLeDa7Ekll/CUsSKwQzpqrP9IJP6p3C1KGnw272wlNkxlx8RH3ODn/mGjFVHRGdLqZMKHc1cICqLg/7U4BzVPVd3XyGpOZIRH4JbANUMmMR/AF5x9WfVzIpKWmcKZm9AZfXjEXw0VMAvGYsgr8a2EtV/xpTv6TxKxG5jJKcjCka5+Aoxoh5y4jIvaq6RWZ/ArAkeyxpbMtrxpJ5S8qTiOzEyCB05/aLTxoMpWmcg6931sl3Odf7fmxqSl3809i0l+tJ014GSl4zFmHeTg6Pe2NRIX8W9mcCf/bWN6nR2qhl2oL+DEztcZnrROQqbCowWGCOlAh7gOQ1aVV5EblQVfcVkcXk55XNy2WbNAYlIucBmwCLgJXhsAK5ZqxuPmlwlMzegMtrxkZ5oa7XjHn5S8KWNHjymjEXryGHm4h8V1W3yzx1mYgs6K7qSQ1TjBFzlVHVT4f1fm8Jh85U1cLANEljR14zFmHejgyPKen14Gs74PVafcpd3XzSgChN40zqm9K0l6RuJSIL2sxY7rEu+HuBPVR1WdifAVyRpt8NltqM2I1VjFhMmaTBk4hMUdXlIjIt7/n2VEZevuR9JwAzNeRMSxr7EpGLgCPaZg30jU8aHKWRvaS+qa5pLy2JyIPk3zlNkacGR2uJyMZtZmytHvJHY7ndlmGBQaYBn+hN1ZOaohBF0xUZs0oZEVlB+fThroLMJPVfrY5zjqmbgM0ceLgbXkTWAQ4DNgRmA9cAnwY+C9wFJLM3OFof+J2I3Mbw2UtFa8Tr5pMGRMnsJfVNXjMWYd6yozWTgA8CkwvYpLEprxlz8ao6J6Ro2Dwcus+RFyupwYoxYt4yqrp2KHc8sBw4D2t3+wNToiqe1Ch5zViEeTsPeBz4LZYj7YtYG9pLVRfV8qGS+qVjG8YnDYjSNM6kvklE1svsrjJjqvrVXvAFr7FQVf8upr5JzVRIvF3ZjEXwKXrZAKvIiJWdV7xlROQuVd2m07GksScRuZQhM/ZOLC2HAEfmmbEIfrGqbhX+noi1u6kpXUxSUlJVJbOX1Ch5zVgZLyLZfIETsJG+T6YO1mCpztDTRdHLUkTXwVGMEfOWEZF5wOnAv2MjgzOBw1R1p64/QFJf5TVjEfyw3LfeXLhJzZeI3Kyqu+TMHCjKQVornzR4StM4k/qmAjNW2Ca9PPDdzN8vAA8B+7ormtRYjULo6RS9bPD1tIjsz3Aj9nR5EXeZ/YDvh02BW8KxpLGv51t/qOpKEXmkw6ibl99GRJ4KfwuwRthPHfUBkaruEh7XbgKfNHhKI3tJfVNIodBSy4ydrKpLe8EnDb5CtMzKZiyCT9HLBlwiMh0zYTszZMSOUtWHelkmaTAlIisZMvoCrIHlgy0aZXHxSYMvEZkEHApsCtwNnKWqL/SLTxo8JbOXNLAKa7P2YeSUva/3q05JvdUohKq+HtgWSNHLktwSkWNU9UQROZX84FJpOnBS0jiXiPwcG/G9CfgH4GFVPbJffNLgKU3jTOqbvGYswrxdCjwJLCTTUU8aKNUdevrYHtUzqWGKMWIRZe4Njwt6UeekpKSB1Osz6zhnYTcX+8knDZiS2Uvqp7xmzMu/RlXfHV+9pDGgY+vkVfUG5+snjR3FGDFXGVW9LDy6coQmJSWNK2XXcb4gIv3mkwZMaRpnUt8kIveo6pY18mcCp6rq4qgKJo1bleRTS+tqkipLRGaXPZ+mAyclJdW97jOtE01KI3tJ/dQ8EdnKYcYq8SJyD/Ai1r4PDAm0n2XoxLZ1V7VO6ru8ZszLp6hlg68YIxZRZkfgD8AFwK1Ye0tKSkpaJVWd2CQ+afCURvaSRl1tZuy1QKkZi+Afx4Jq5EpVH+7Zh0lKShqTEpHHKDFieVN4vWVCHrXdsNQMWwO/Bi5Q1SU9+yBJSUlJSUklSmYvadTlNWMRfEo6m5SUVKoYI9aNeQsBpmYCJwHHqeppXX+IpKSkpKSkDkpmL2nU5TVjEfwjwClFz6tq4XNJSUnjTzFGrGqZwO0R2OnAbCzP1R97U/ukpKSkpKRipTV7Sf3QBiLymaInc8yYl58IvIy0PiYpKalEOUbsB8DFvSojIucCWwJXYIbwnh5VPSkpKSkpqZKS2Uvqh7xmzMsvT4nTk5KSyhRjxCLKfBiLgnckcEQm5HmKgpeUlJSUNCpK0ziTRl2jMI3zTlV9Q1ztkpKSxoNE5EWGwpFnL4SFRiymTFJSUlJSUj+VRvaS+iHv9Eov/04nn5SUNM6kqhNGo0xSUlJSUlI/lUb2kkZdIjJZVf+3Lj4pKSkpKSkpKSkpKZm9pKSkpKSkpKSkpKSkgVSakpKUlJSUlJSUlJSUlDSASmYvKSkpKSkpKSkpKSlpAJXMXlJSUlJSUlJSUlJS0gAqmb2kpKSkpKSkpKSkpKQBVDJ7SUlJSUlJSUlJSUlJA6j/D0YThp33ZNqmAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1080x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "error = labels - preds\n",
    "sq_error = error * error\n",
    "avg_error = np.mean(sq_error, axis=0)\n",
    "plt.figure(figsize=(15, 10))\n",
    "plt.xticks(rotation=90)\n",
    "plt.ylabel('Prediction Error (mm)')\n",
    "bar = plt.bar(df.columns[8:], avg_error)\n",
    "for i in range(0,63,3):\n",
    "    bar[i].set_color('coral')\n",
    "    bar[i+1].set_color('olivedrab')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training the Model On Tips Only\n",
    "In an attempt to improve the accuracy of prediction of the fingertips the above models are modified and trained only using the fingertip position data to understand if this simplification yields an improvement. This is done on both of the architectures tested above."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Autoencoder Architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoding_dim = 9 #dimensionality of 'feature' vector\n",
    "\n",
    "a_fn = None\n",
    "\n",
    "# Full autoencoder\n",
    "input_vec = Input(shape=(18,))\n",
    "dense_0 = Dense(32, activation=a_fn)(input_vec)\n",
    "dense_1 = Dense(16, activation=a_fn)(dense_0)\n",
    "encoded = Dense(encoding_dim, activation=a_fn)(dense_1)\n",
    "dense_2 = Dense(32, activation=a_fn, name='decoder_0')(encoded)\n",
    "dense_3 = Dense(16, activation=a_fn, name='decoder_1')(dense_2)\n",
    "decoded = Dense(18, activation=a_fn, name='decoder_output')(dense_3)\n",
    "\n",
    "autoencoder = Model(input_vec, decoded)\n",
    "\n",
    "# Encoder from autoencoder\n",
    "encoder = Model(input_vec, encoded)\n",
    "\n",
    "# Decoder from autoencoder layers\n",
    "decoder_input = Input(shape=(encoding_dim,), name='encoded_input')\n",
    "decode_0 = autoencoder.layers[-3](decoder_input)\n",
    "decode_1 = autoencoder.layers[-2](decode_0)\n",
    "decode_output = autoencoder.layers[-1](decode_1)\n",
    "decoder = Model(decoder_input, decode_output, name='decoder')\n",
    "\n",
    "# Train Autoencoder\n",
    "optimizer = tf.keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)\n",
    "autoencoder.compile(optimizer=optimizer, loss='mean_squared_error')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16481, 18)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_ar_t = label_ar[:,[0,1,2,12,13,14,24,25,26,36,37,38,48,49,50,60,61,62]]\n",
    "label_ar_t.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 13184 samples, validate on 3297 samples\n",
      "Epoch 1/30\n",
      "13184/13184 [==============================] - 2s 125us/sample - loss: 1804.8644 - val_loss: 844.6304\n",
      "Epoch 2/30\n",
      "13184/13184 [==============================] - 0s 6us/sample - loss: 363.1596 - val_loss: 143.7439\n",
      "Epoch 3/30\n",
      "13184/13184 [==============================] - 0s 6us/sample - loss: 108.8713 - val_loss: 135.5584\n",
      "Epoch 4/30\n",
      "13184/13184 [==============================] - 0s 6us/sample - loss: 84.1743 - val_loss: 121.3580\n",
      "Epoch 5/30\n",
      "13184/13184 [==============================] - 0s 6us/sample - loss: 70.7524 - val_loss: 107.9477\n",
      "Epoch 6/30\n",
      "13184/13184 [==============================] - 0s 7us/sample - loss: 59.9721 - val_loss: 100.2388\n",
      "Epoch 7/30\n",
      "13184/13184 [==============================] - 0s 6us/sample - loss: 51.1394 - val_loss: 91.6366\n",
      "Epoch 8/30\n",
      "13184/13184 [==============================] - 0s 6us/sample - loss: 44.0869 - val_loss: 85.8753\n",
      "Epoch 9/30\n",
      "13184/13184 [==============================] - 0s 6us/sample - loss: 38.6877 - val_loss: 78.2946\n",
      "Epoch 10/30\n",
      "13184/13184 [==============================] - 0s 6us/sample - loss: 34.5467 - val_loss: 72.4935\n",
      "Epoch 11/30\n",
      "13184/13184 [==============================] - 0s 6us/sample - loss: 31.3283 - val_loss: 68.1716\n",
      "Epoch 12/30\n",
      "13184/13184 [==============================] - 0s 7us/sample - loss: 28.8070 - val_loss: 62.5642\n",
      "Epoch 13/30\n",
      "13184/13184 [==============================] - 0s 6us/sample - loss: 26.7931 - val_loss: 59.0995\n",
      "Epoch 14/30\n",
      "13184/13184 [==============================] - 0s 7us/sample - loss: 25.1587 - val_loss: 57.0017\n",
      "Epoch 15/30\n",
      "13184/13184 [==============================] - 0s 7us/sample - loss: 23.7901 - val_loss: 54.6618\n",
      "Epoch 16/30\n",
      "13184/13184 [==============================] - 0s 7us/sample - loss: 22.5898 - val_loss: 52.8294\n",
      "Epoch 17/30\n",
      "13184/13184 [==============================] - 0s 7us/sample - loss: 21.5261 - val_loss: 50.7334\n",
      "Epoch 18/30\n",
      "13184/13184 [==============================] - 0s 7us/sample - loss: 20.5708 - val_loss: 49.1961\n",
      "Epoch 19/30\n",
      "13184/13184 [==============================] - 0s 7us/sample - loss: 19.6828 - val_loss: 47.7905\n",
      "Epoch 20/30\n",
      "13184/13184 [==============================] - 0s 8us/sample - loss: 18.8381 - val_loss: 46.5224\n",
      "Epoch 21/30\n",
      "13184/13184 [==============================] - ETA: 0s - loss: 18.16 - 0s 7us/sample - loss: 18.0371 - val_loss: 46.3079\n",
      "Epoch 22/30\n",
      "13184/13184 [==============================] - 0s 6us/sample - loss: 17.2669 - val_loss: 44.2411\n",
      "Epoch 23/30\n",
      "13184/13184 [==============================] - 0s 8us/sample - loss: 16.5206 - val_loss: 43.1075\n",
      "Epoch 24/30\n",
      "13184/13184 [==============================] - 0s 8us/sample - loss: 15.7940 - val_loss: 42.0993\n",
      "Epoch 25/30\n",
      "13184/13184 [==============================] - 0s 7us/sample - loss: 15.0827 - val_loss: 41.1903\n",
      "Epoch 26/30\n",
      "13184/13184 [==============================] - 0s 7us/sample - loss: 14.4066 - val_loss: 39.9729\n",
      "Epoch 27/30\n",
      "13184/13184 [==============================] - 0s 7us/sample - loss: 13.7648 - val_loss: 37.9617\n",
      "Epoch 28/30\n",
      "13184/13184 [==============================] - 0s 7us/sample - loss: 13.1585 - val_loss: 37.1820\n",
      "Epoch 29/30\n",
      "13184/13184 [==============================] - 0s 7us/sample - loss: 12.5901 - val_loss: 36.5573\n",
      "Epoch 30/30\n",
      "13184/13184 [==============================] - 0s 9us/sample - loss: 12.0513 - val_loss: 35.0339\n"
     ]
    }
   ],
   "source": [
    "ret = autoencoder.fit(label_ar_t, label_ar_t, batch_size=512, epochs=30, verbose=1, validation_split=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a smaller labels vector with only the tip and wrist x,y,z:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(89004, 18)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels_tip = labels[:,[0,1,2,12,13,14,24,25,26,36,37,38,48,49,50,60,61,62]]\n",
    "labels_tip.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_v1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "inputs (InputLayer)          [(None, None, 8)]         0         \n",
      "_________________________________________________________________\n",
      "lstm_0 (LSTM)                (None, None, 64)          18688     \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, None, 64)          0         \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 64)                33024     \n",
      "_________________________________________________________________\n",
      "lstm_out (Dense)             (None, 9)                 585       \n",
      "_________________________________________________________________\n",
      "decoder_0 (Dense)            (None, 32)                320       \n",
      "_________________________________________________________________\n",
      "decoder_1 (Dense)            (None, 16)                528       \n",
      "_________________________________________________________________\n",
      "decoder_output (Dense)       (None, 18)                306       \n",
      "=================================================================\n",
      "Total params: 53,451\n",
      "Trainable params: 53,451\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "#lstm layers\n",
    "inputs = Input(shape=(None, 8), name=\"inputs\")\n",
    "lstm_0 = LSTM(64, return_sequences=True, name=\"lstm_0\")(inputs)\n",
    "do = Dropout(0.2)(lstm_0)\n",
    "lstm_1 = LSTM(64, return_sequences=False, name=\"lstm_1\")(do)\n",
    "lstm_out = Dense(9, activation=None, name=\"lstm_out\")(lstm_1)\n",
    "\n",
    "#decoder layers\n",
    "decoder_0 = decoder.get_layer(\"decoder_0\")(lstm_out)\n",
    "decoder_0.trainable = False\n",
    "decoder_1 = decoder.get_layer(\"decoder_1\")(decoder_0)\n",
    "decoder_1.trainable = False\n",
    "decoder_output = decoder.get_layer(\"decoder_output\")(decoder_1)\n",
    "decoder_output.trainable = False\n",
    "\n",
    "model = Model(inputs, decoder_output, name=\"model_v1\")\n",
    "optimizer = tf.keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.00, amsgrad=False)\n",
    "model.compile(optimizer=optimizer, loss='mse')\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 71203 samples, validate on 17801 samples\n",
      "Epoch 1/4\n",
      "71203/71203 [==============================] - 88s 1ms/sample - loss: 371.7387 - val_loss: 296.5638\n",
      "Epoch 2/4\n",
      "71203/71203 [==============================] - 81s 1ms/sample - loss: 258.2415 - val_loss: 275.9274\n",
      "Epoch 3/4\n",
      "71203/71203 [==============================] - 77s 1ms/sample - loss: 218.4306 - val_loss: 257.8691\n",
      "Epoch 4/4\n",
      "71203/71203 [==============================] - 82s 1ms/sample - loss: 194.0974 - val_loss: 247.3258\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(features, labels_tip, batch_size=seq_length, epochs=4, verbose=1, validation_split=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visual Model Error\n",
    "Model error for the reduced Autoencoder architecture:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = model.predict(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3sAAAJ2CAYAAADmJJuJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzde7htZ1ke/PshgRJoJEAi8kEggEHECgoRAljk8ElRjqXiGULEgpUCVkXBVvFQC3JSsELhI2DwWKRYjuUgJ7EYJBwDAhoiVFKQICaNpJyf7485dlhZWXvvlb3X3GOOd/9+1zWvNcc75tq5WXOy1rznGON9q7sDAADAWK42dwAAAAD2nrIHAAAwIGUPAABgQMoeAADAgJQ9AACAASl7AAAAA1pr2auqj1bVeVX1nqo6dxq7XlW9vqr+evp63Wm8qupZVXV+Vb2vqm63zmwAAAAjOxJH9u7e3d/S3adN249P8obuPjXJG6btJPmuJKdOt0ckec4RyAYAADCkOU7jfECSs6f7Zyd54JbxF/XKOUlOqKobzpAPAABg8Y5d87/fSV5XVZ3kud39vCQ36O5PTPs/meQG0/0bJfnbLd/78WnsE9mPE088sU855ZQ9Dw0AALAE73znOz/d3SfttG/dZe/bu/vCqvraJK+vqg9t3dndPRXBXauqR2R1mmducpOb5Nxzz927tAAAAAtSVR/b3761nsbZ3RdOXz+V5I+T3CHJ3+07PXP6+qnp4RcmOXnLt994Gtv+bz6vu0/r7tNOOmnHAgsAAHDUW1vZq6prV9Xx++4nuVeS9yd5eZIzpoedkeRl0/2XJ3noNCvn6Uku2XK6JwAAAFfBOk/jvEGSP66qff+d3+/u11TVO5K8uKoenuRjSb53evyrk3x3kvOTXJbkzDVmAwAAGNrayl53X5DktjuM/32Se+4w3kketa48AAAAR5M5ll4AAABgzZQ9AACAASl7AAAAA1L2AAAABqTsAQAADEjZAwAAGJCyBwAAMCBlDwAAYEDKHgAAwICUPQAAgAEpewAAAANS9gAAAAak7AEAAAxI2QMAABiQsgcAADAgZQ8AAGBAyh4AAMCAlD0AAIABKXsAAAADUvYAAAAGdOzcAQCYxxOee3ouvezTc8dIkhx/rRPzpEeeM3cMABiKI3sAR6lNKXrJZmUBgFEoewAAAANS9gAAAAak7AEAAAxI2QMAABiQsgcAADAgZQ8AAGBAyh4AAMCAlD0AAIABKXsAAAADUvYAAAAGpOwBAAAMSNkDAAAYkLIHAAAwIGUPAABgQMoeAADAgJQ9AACAASl7AAAAA1L2AAAABqTsAQAADEjZAwAAGJCyBwAAMCBlDwAAYEDKHgAAwICUPQAAgAEpewAAAANS9gAAAAak7AEAAAxI2QMAABiQsgcAADAgZQ8AAGBAyh4AAMCAlD0AAIABKXsAAAADUvYAAAAGpOwBAAAMSNkDAAAYkLIHAAAwIGUPAABgQMoeAADAgJQ9AACAASl7AAAAA1L2AAAABqTsAQAADEjZAwAAGJCyBwAAMCBlDwAAYEDKHgAAwICUPQAAgAEdO3cAAADgyHrCc0/PpZd9eu4YOf5aJ+ZJjzxn7hjDcmQPAACOMptQ9JLNyTEqZQ8AAGBAyh4AAMCAlD0AAIABKXsAAAADUvYAAAAGpOwBAAAMSNkDAAAYkLIHAAAwIGUPAABgQMoeAADAgJQ9AACAASl7AAAAA1L2AAAABqTsAQAADEjZAwAAGJCyBwAAMCBlDwAAYEDKHgAAwICUPQAAgAEpewAAAANS9gAAAAak7AEAAAxI2QMAABiQsgcAADAgZQ8AAGBAyh4AAMCAlD0AAIABKXsAAAADWnvZq6pjqurdVfXKaftmVfX2qjq/qv5rVV1jGv8n0/b50/5T1p0NAABgVEfiyN5jk3xwy/avJfn17v76JP+Q5OHT+MOT/MM0/uvT4wAAADgEay17VXXjJPdJ8vxpu5LcI8lLpoecneSB0/0HTNuZ9t9zejwAAABX0bqP7P1Gkp9J8pVp+/pJLu7uL03bH09yo+n+jZL8bZJM+y+ZHg8AAMBVtLayV1X3TfKp7n7nHv+7j6iqc6vq3Isuumgv/2kAAIBhrPPI3l2S3L+qPprkD7M6ffOZSU6oqmOnx9w4yYXT/QuTnJwk0/7rJPn77f9odz+vu0/r7tNOOumkNcYHAABYrrWVve5+QnffuLtPSfL9Sd7Y3T+U5E1Jvmd62BlJXjbdf/m0nWn/G7u715UPAABgZHOss/ezSX6yqs7P6pq8s6bxs5Jcfxr/ySSPnyEbAADAEI49+EMOX3e/Ocmbp/sXJLnDDo/5XJIHH4k8AAAAo5vjyB4AAABrpuwBAAAMSNkDAAAYkLIHAAAwIGUPAABgQMoeAADAgJQ9AACAASl7AAAAA1L2AAAABqTsAQAADEjZAwAAGJCyBwAAMCBlDwAAYEDKHgAAwICUPQAAgAEpewAAAANS9gAAAAak7AEAAAxI2QMAABiQsgcAADCgY+cOAMznCc89PZde9um5Y+T4a52YJz3ynLljAAAMxZE9OIptQtFLNicHAMBIlD0AAIABKXsAAAADUvYAAAAGpOwBAAAMSNkDAAAYkLIHAAAwIOvsAcBRylqbAGNzZA8AjlKbUPSSzckBMBplDwAAYEDKHgAAwICUPQAAgAEpewAAAANS9gAAAAak7AEAAAxI2QMAABiQsgcAADAgZQ8AAGBAyh4AAMCAlD0AAIABKXsAAAADUvYAAAAGpOwBAAAMSNkDAAAYkLIHAAAwIGUPAABgQMoeAADAgJQ9AACAASl7AAAAA1L2AAAABqTsAQAADOjYuQPAcJ56ZvLZi+dOsXLtE5LHvXDuFAAAzMCRPdhrm1L0ks3KAgDAEaXsAQAADEjZAwAAGJCyBwAAMCATtAAAHIWe8NzTc+lln547Ro6/1ol50iPPmTsGDMmRPQCAo9AmFL1kc3LAiJQ9AACAASl7AAAAA1L2AAAABqTsAQAADEjZAwAAGJCyBwAAMCBlDwAAYEDKHgAAwICUPQAAgAEpewAAAANS9gAAAAak7AEAAAxI2QMAABiQsgcAADAgZQ8AAGBAyh4AAMCAlD0AAIABKXsAAAADUvYAAAAGpOwBAAAMSNkDAAAYkLIHAAAwIGUPAABgQMoeAADAgJQ9AACAASl7AAAAA1L2AAAABqTsAQAADEjZAwAAGJCyBwAAMCBlDwAAYEDKHgAAwICUPQAAgAEpewAAAANS9gAAAAak7AEAAAxI2QMAABiQsgcAADAgZQ8AAGBAyh4AAMCA1lb2quqaVfUXVfXeqvpAVf3SNH6zqnp7VZ1fVf+1qq4xjf+Tafv8af8p68oGAAAwunUe2ft8knt0922TfEuSe1fV6Ul+Lcmvd/fXJ/mHJA+fHv/wJP8wjf/69DgAAAAOwdrKXq/847R59enWSe6R5CXT+NlJHjjdf8C0nWn/Pauq1pUPAABgZGu9Zq+qjqmq9yT5VJLXJ/lIkou7+0vTQz6e5EbT/Rsl+dskmfZfkuT668wHAAAwqrWWve7+cnd/S5IbJ7lDklsd7r9ZVY+oqnOr6tyLLrrosDMCAACMaNdlr6quW1XfVFU3r6qrVBK7++Ikb0pypyQnVNWx064bJ7lwun9hkpOn/9axSa6T5O93+Lee192ndfdpJ5100lWJAQAAcNQ4YGmrqutU1c9V1XlJzkny3CQvTvKxqvqjqrr7Ab73pKo6Ybp/XJLvTPLBrErf90wPOyPJy6b7L5+2M+1/Y3f3of3PAgAAOLode5D9L0nyoiT/fDo6d7mqun2Sh1TVzbv7rB2+94ZJzq6qY7IqlS/u7ldW1V8m+cOq+o9J3p1k3/eeleR3qur8JJ9J8v2H/L8KAADgKHfAstfd33mAfe9M8s4D7H9fkm/dYfyCrK7f2z7+uSQPPlAeAAAAdudgR/YuV1W3SXLK1u/p7peuIRMAAACHaVdlr6pekOQ2ST6Q5CvTcCdR9gAAADbQbo/snd7dt15rEgAAAPbMbpdQ+POqUvYAAAAWYrdH9l6UVeH7ZJLPJ6kk3d23WVsyAAAADtluy95ZSR6S5Lx89Zo9AAAANtRuy95F3f3ytSYBAABgz+y27L27qn4/ySuyOo0ziaUXAAAANtVuy95xWZW8e20Zs/QCAADAhtpV2evuM9cdBAAAgL2z20XVb5bk0UlO2fo93X3/9cQCAADgcOz2NM7/ntWMnK+I2TgBAAA23m7L3ue6+1lrTQIAAMCe2W3Ze2ZVPTHJ63LF2TjftZZUAAAAHJbdlr1vzmpR9Xvkq6dx9rQNAADAhtlt2Xtwkpt39xfWGQYAAIC9cbVdPu79SU5YZxAAAAD2zm6P7J2Q5ENV9Y5c8Zo9Sy8AAABsoN2WvSeuNQUAAAB76oBlr6qqV95ysMfsfTQAAAAO1cGu2XtTVT26qm6ydbCqrlFV96iqs5Ocsb54AAAAHIqDncZ57yQ/kuQPqupmSS5OclxWJfF1SX6ju9+93ogAAABcVQcse939uSTPTvLsqrp6khOT/N/uvvhIhAMAAODQ7HaClnT3F5N8Yo1ZAAAA2CO7XWcPAACABVH2AAAABnTQsldVx1TVm45EGAAAAPbGQcted385yVeq6jpHIA8AAAB7YLcTtPxjkvOq6vVJPrtvsLsfs5ZUAAAAHJbdlr2XTjcAAAAWYFdlr7vPrqprJLnlNPThaSkGAAAANtCuyl5V3S3J2Uk+mqSSnFxVZ3T3n64vGgAAAIdqt6dxPj3Jvbr7w0lSVbdM8gdJbr+uYAAAABy63a6zd/V9RS9Juvuvklx9PZEAAAA4XLs9snduVT0/ye9O2z+U5Nz1RAIAAOBw7bbs/Zskj0qyb6mFtyZ59loSAQAAcNgOWvaq6pgkL+juH0ryjPVHAgAA4HAd9Jq97v5ykptOSy8AAACwALs9jfOCJP+zql6e5LP7BrvbkT4AAIANtNuy95HpdrUkx68vDgAAAHtht9fsHd/dP30E8gAAALAHdnvN3l2OQBYAAAD2yG5P43zPdL3eH+WK1+y9dC2pAAAAOCy7LXvXTPL3Se6xZayTKHsAAAAbaFdlr7vPXHcQAAAA9s4Br9mrqhdvuf9r2/a9bl2hAAAAODwHm6Dl1C33v3PbvpP2OAsAAAB75GBlrw9xHwAAADM62DV716qqb82qFB433a/pdty6wwEAAHBoDlb2PpHkGdP9T265v28bAACADXTAstfddz9SQQAAANg7B7tmDwAAgAVS9gAAAAak7AEAAAzoYBO0XK6qbpTkplu/p7v/dB2hAAAAODy7KntV9WtJvi/JXyb58jTcSZQ9AACADbTbI3sPTPIN3f35dYYBAABgb+z2mr0Lklx9nUEAAADYO7s9sndZkvdU1RuSXH50r7sfs5ZUAAAAHJbdlr2XTzcAAAAWYFdlr7vPrqprJLnlNPTh7v7i+mIBAABwOHY7G+fdkpyd5KNJKsnJVXWGpRcAAAA2025P43x6knt194eTpKpumeQPktx+XcEAAAA4dLudjfPq+4peknT3X8XsnAAAABtrt0f2zq2q5yf53Wn7h5Kcu55IAAAAHK7dlr1/k+RRSfYttfDWJM9eSyIAAAAO225n4/x8kmdMNwAAADbcActeVb24u7+3qs5L0tv3d/dt1pYMAACAQ3awI3uPnb7ed91BAAAA2DsHnI2zuz8x3f3x7v7Y1luSH19/PAAAAA7Fbpde+M4dxr5rL4MAAACwdw52zd6/yeoI3i2q6n1bdh2f5G3rDAYAAMChO9g1e7+f5H8keVKSx28Zv7S7P7O2VAAAAByWg12zd0l3fzTJM5N8Zsv1el+qqjseiYAAAABcdbu9Zu85Sf5xy/Y/TmMAAABsoN2Wveruy9fZ6+6vZJcLsgMAAHDk7bbsXVBVj6mqq0+3xya5YJ3BAAAAOHS7LXs/luTOSS5M8vEkd0zyiHWFAgAA4PDs6lTM7v5Uku9fcxYAAAD2yMHW2fuZ7n5KVf1mkt6+v7sfs7ZkAAAAHLKDHdn74PT13HUHAQAAYO8csOx19yumr2cfmTgAAADshYOdxvmK7HD65j7dff89TwQAAMBhO9hpnE+bvj4oydcl+d1p+weS/N26QgEAAHB4DnYa51uSpKqe3t2nbdn1iqpyHR8AAMCG2u06e9euqpvv26iqmyW59noiAQAAcLh2tc5ekn+X5M1VdUGSSnLTJI9cWyoAAAAOy24XVX9NVZ2a5FbT0Ie6+/PriwUAAMDh2NVpnFV1rSSPS/Jvu/u9SW5SVfddazIAAAAO2W6v2Xthki8kudO0fWGS/7iWRAAAABy23Za9W3T3U5J8MUm6+7Ksrt0DAABgA+227H2hqo7LtMB6Vd0iiWv2AAAANtRuZ+N8YpLXJDm5qn4vyV2SPGxdoQAAADg8By17VVVJPpTkQUlOz+r0zcd296fXnA0AAIBDdNCy191dVa/u7m9O8qojkAkAAIDDtNtr9t5VVd+21iQAAADsmd1es3fHJD9cVR9N8tmsTuXs7r7NuoIBAABw6HZb9v7FWlMAAACwpw5Y9qrqmkl+LMnXJzkvyVnd/aUjEQwAAIBDd7Br9s5OclpWRe+7kjx97YkAAAA4bAcre7fu7h/u7ucm+Z4k/3y3/3BVnVxVb6qqv6yqD1TVY6fx61XV66vqr6ev153Gq6qeVVXnV9X7qup2h/y/CgAA4Ch3sLL3xX13DuH0zS8l+anuvnVW6/M9qqpuneTxSd7Q3acmecO0nayOHJ463R6R5DlX8b8HAADA5GATtNy2qv7PdL+SHDdt75uN82v2943d/Ykkn5juX1pVH0xyoyQPSHK36WFnJ3lzkp+dxl/U3Z3knKo6oapuOP07AAAAXAUHLHvdfcxe/Eeq6pQk35rk7UlusKXAfTLJDab7N0ryt1u+7ePTmLIHAABwFe12UfVDVlX/NMl/S/IT3f1/tu6bjuL1Vfz3HlFV51bVuRdddNEeJgUAABjHWsteVV09q6L3e9390mn476rqhtP+Gyb51DR+YZKTt3z7jaexK+ju53X3ad192kknnbS+8AAAAAu2trJXVZXkrCQf7O5nbNn18iRnTPfPSPKyLeMPnWblPD3JJa7XAwAAODQHm6DlcNwlyUOSnFdV75nGfi7Jk5O8uKoenuRjSb532vfqJN+d5PwklyU5c43ZAAAAhra2stfdf5bVrJ07uecOj+8kj1pXHgAAgKPJ2idoAQAA4MhT9gAAAAak7AEAAAxI2QMAABiQsgcAADAgZQ8AAGBAyh4AAMCAlD0AAIABKXsAAAADUvYAAAAGpOwBAAAMSNkDAAAYkLIHAAAwIGUPAABgQMoeAADAgJQ9AACAASl7AAAAA1L2AAAABqTsAQAADEjZAwAAGJCyBwAAMCBlDwAAYEDKHgAAwICUPQAAgAEpewAAAANS9gAAAAak7AEAAAxI2QMAABiQsgcAADAgZQ8AAGBAyh4AAMCAlD0AAIABKXsAAAADUvYAAAAGpOwBAAAMSNkDAAAYkLIHAAAwIGUPAABgQMoeAADAgJQ9AACAASl7AAAAA1L2AAAABnTs3AEAduOUx79q7ghJko8++T5zRwAA2BVH9gAAAAak7AEAAAxI2QMAABiQsgcAADAgZQ8AAGBAyh4AAMCAlD0AAIABKXsAAAADUvYAAAAGpOwBAAAMSNkDAAAYkLIHAAAwIGUPAABgQMoeAADAgJQ9AACAASl7AAAAA1L2AAAABqTsAQAADEjZAwAAGJCyBwAAMCBlDwAAYEDKHgAAwICUPQAAgAEdO3cAAAAYwlPPTD578dwpVq59QvK4F86dgpk5sgcAAHthU4pesllZmI2yBwAAMCBlDwAAYEDKHgAAwICUPQAAgAEpewAAAANS9gAAAAak7AEAAAxI2QMAABiQsgcAADAgZQ8AAGBAyh4AAMCAlD0AAIABKXsAAAADUvYAAAAGpOwBAAAMSNkDAAAYkLIHAAAwIGUPAABgQMoeAADAgJQ9AACAASl7AAAAA1L2AAAABnTs3AEAhvPUM5PPXjx3iuTaJySPe+HcKQDgsJzy+FfNHSFJ8tEn32fuCFeZI3sAe20Til6yOTkAgFkoewAAAANS9gAAAAbkmj0ANp7rNQDgqlP2AGAvmaAHgA3hNE4A2EubUPSSzckBwGwc2QMA2CuO7AIbxJE9AIC9sglFL9mcHMCs1lb2quoFVfWpqnr/lrHrVdXrq+qvp6/Xncarqp5VVedX1fuq6nbrygUAAHA0WOeRvd9Ocu9tY49P8obuPjXJG6btJPmuJKdOt0ckec4acwEAAAxvbWWvu/80yWe2DT8gydnT/bOTPHDL+It65ZwkJ1TVDdeVDQAAYHRH+pq9G3T3J6b7n0xyg+n+jZL87ZbHfXwaAwAA4BDMNkFLd3eSvqrfV1WPqKpzq+rciy66aA3JAAAAlu9Il72/23d65vT1U9P4hUlO3vK4G09jV9Ldz+vu07r7tJNOOmmtYQEAAJbqSJe9lyc5Y7p/RpKXbRl/6DQr5+lJLtlyuicAAABX0doWVa+qP0hytyQnVtXHkzwxyZOTvLiqHp7kY0m+d3r4q5N8d5Lzk1yW5Mx15QIAADgarK3sdfcP7GfXPXd4bCd51LqyAAAAHG1mm6AFAACA9VH2AAAABqTsAQAADEjZAwAAGNDaJmgBAIC9cMrjXzV3hCTJR598n7kjwFXiyB4AAMCAlD0AAIABKXsAAAADUvYAAAAGpOwBAAAMSNkDAAAYkLIHAAAwIGUPAABgQMoeAADAgJQ9AACAASl7AAAAA1L2AAAABnTs3AEAAA7klMe/au4Il/vok+8zdwSAXXNkDwAAYEDKHgAAwICUPQAAgAEpewAAAANS9gAAAAak7AEAAAxI2QMAABiQsgcAADAgZQ8AAGBAyh4AAMCAlD0AAIABKXsAAAADUvYAAAAGpOwBAAAMSNkDAAAYkLIHAAAwIGUPAABgQMoeAADAgJQ9AACAASl7AAAAA1L2AAAABqTsAQAADEjZAwAAGJCyBwAAMCBlDwAAYEDKHgAAwICOnTsA83rCc0/PpZd9eu4YOf5aJ+ZJjzxn7hgAADAMR/aOcptQ9JLNyQEAAKNQ9gAAAAak7AEAAAxI2QMAABiQsgcAADAgZQ8AAGBAyh4AAMCAlD0AAIABKXsAAAADUvYAAAAGpOwBAAAMSNkDAAAYkLIHAAAwIGUPAABgQMoeAADAgJQ9AACAASl7AAAAA1L2AAAABqTsAQAADEjZAwAAGJCyBwAAMCBlDwAAYEDKHgAAwICUPQAAgAEpewAAAANS9gAAAAak7AEAAAxI2QMAABiQsgcAADAgZQ8AAGBAyh4AAMCAlD0AAIABKXsAAAADUvYAAAAGpOwBAAAMSNkDAAAYkLIHAAAwoGPnDjCkp56ZfPbiuVMk1z4hedwL504BAADMwJG9ddiEopdsTg4AAOCIU/YAAAAGpOwBAAAMSNkDAAAYkLIHAAAwIGUPAABgQMoeAADAgJQ9AACAASl7AAAAA1L2AAAABqTsAQAADEjZAwAAGJCyBwAAMKBj5w4Au3XK4181d4QkyUeffJ+5IwAAwEE5sgcAADAgZQ8AAGBAyh4AAMCANqrsVdW9q+rDVXV+VT1+7jwAAABLtTFlr6qOSfJbSb4rya2T/EBV3XreVAAAAMu0MWUvyR2SnN/dF3T3F5L8YZIHzJwJAABgkTap7N0oyd9u2f74NAYAAMBVVN09d4YkSVV9T5J7d/ePTtsPSXLH7v632x73iCSPmDa/IcmHj2jQI+fEJJ+eO8Quybr3lpIzkXVdlpJ1KTkTWddlKVmXkjORdV2WknUpORNZN8VNu/uknXZs0qLqFyY5ecv2jaexK+ju5yV53pEKNZeqOre7T5s7x27IuveWkjORdV2WknUpORNZ12UpWZeSM5F1XZaSdSk5E1mXYJNO43xHklOr6mZVdY0k35/k5TNnAgAAWKSNObLX3V+qqn+b5LVJjknygu7+wMyxAAAAFmljyl6SdPerk7x67hwbYkmnqsq695aSM5F1XZaSdSk5E1nXZSlZl5IzkXVdlpJ1KTkTWTfexkzQAgAAwN7ZpGv2AAAA2CPKHgBsoGmyso1TVdefOwMAu6PsbYiq+n93GDtjjiwHU1VPr6pvmjvHblTVR6rqx7aNvXKuPPtTVbfeYexuM0Q5qKp6dFVdd+4cu1FVb6iq7942tpHn7C/lNeD5X4+qenNVnbJl+w5ZzVK9ic6pqj+qqu+uqpo7zP5U1UOq6vhtY/edK8+BVNXX7jD2DXNkOZiqul9VLeL9Y1WdVVXfsm3sF2eKs19V9fBt28dU1RPnynMgVfXSqrrPEl4DVfXlqnry1t9TVfWuOTPNYeOfqKPIL1TVc6rq2lV1g6p6RZL7zR1qPz6Y5HlV9faq+rGqus7cgQ7gi0nuXlUv3PIp+Y3mDLQfL66qn62V46rqN5M8ae5Q+3GDJO+oqhdX1b03+c1ekpsl+dltfzQ3dY2dpbwGPP/r8aQkr6mqH6+qX03yX5KcOXOm/bllVhMdPCTJX1fVf6qqW86caSe/meStVfWNW8Z+ea4wB/HWqvrefRtV9VNJ/njGPAfyfVk970+pqlvNHeYg/kWSs6vqoVvG7j9XmAO4Z1W9uqpuOH2Yfk6S4w/2TTN5dpIfzOo18ORN/VBi8oGsus7rqup609gm/81aC2Vvc3xHko8keU+SP0vy+939PfNG2ll3P7+775LkoUlOSfK+qvr9qrr7vMl2dFl3f19WBfWtVXWTJJs4K9Edk5yc5G1ZfZr/v5PcZdZE+9Hd/yHJqUnOSvKwfPXN3i1mDbazi5PcM8kNquoVG/7BxCJeA57/9eju1yb5sSTPTPIjSb67uzfyE+heeX13/0CSf53kjCR/UVVvqao7zRxvq7/J6mf5kqp68DS2qW/07pbkIdMR0z/NqlDfYd5IO+vuH07yrVm9Z/ntqvrzqnrE9qOoG+JTSe6a5MFV9VtVdWw28DXQ3T+Y5Owk52U1K/1PdPdPz5tqZ939J939Q0lul+SjSf6kqt5WVWdW1dXnTXclX+run0ny/KzeA94+m/kecDqf80UAABiISURBVK2Uvc1x3ax+sX8kyeeT3HSTPzGvqmOS3Gq6fTrJe5P8ZFX94azBrqySpLufkuTfJ3ldkhvPmmhnX0zyf5Mcl+SaSf6mu78yb6T969U0vp+cbl/K6vX7kqp6yqzBrqy6+0vd/eNJ/ltWH6Rc6XSpDbGY14Dnf+9V1c9ndSTqrkl+Mcmbq+o+s4baj6q6flU9tqrOTfLTSR6d5MQkP5Xk92cNd0U9FebvSPKIqnpaVuv4bpzu/kSS1yS5U1Yfop7d3f84a6gD6O7/k+QlSf4wyQ2T/Msk76qqR88a7Mqquy/p7vsluSjJm5Ns3Ic+VXVqksdm9XvqY1kV/2vNm2r/anXd7sOS/GiSd2f1IdXtkrx+xlg72fce8L9mdUT6hUluPmuiOXS32wbckvxVkh+Z7h+X5FlJ3jZ3rv1k/fUkf53kuUnusG3fh+fOty3P/bZt3zTJL8yda4ec783q9KKrZ/WH82VJ/mjuXPvJ+tgk70zy2iQPTnL1afxqST4yd75tWR+5bfv2SV4wd64lvwY8/2vL+htJjtuyfdMkr587136y/lWSn09y4x32/ezc+bZkedWW+1dL8tQkX5k7136y/kmSFyU5Ick3J/mLJE+bO9d+st4/q1NMz0vyuCRfO41fK8lH5863Lesvbdu+X5I3zp1rh5wfSnLP6X5l9cHJB+bOtZ+sf5zkL5M8IckNt+07d+582/Lcftv2dZI8dO5cR/pmnb0NUVU36e7/tW3srt39p3Nl2p+qOjPJi7v7szvsu053XzJDrEWrqtO6+9xtYw/p7t+ZK9P+VNUvZfWG+WM77PvG7v7gDLEWbymvAc8/VVXtzcOeqqoHdvd/37J9bJIndPevzBhrR1V1dpKzdnp/UlX37O43zBBr0arqa3p1tHTr2C27+6/myrQ/VXX37n7T3DnYPWUPAABgQK7ZAwAAGNCxcweAdaqq3+nuhxxsjHFV1ddlNflRJ3lHd39y5kgcQZ5/OLpNyy7dKqvfAR/u7i/MHIkjqKoelOTbs3r+/6y7N3VJk7VxZG9DVNWVrsvZaWwTLClrkiss/j7NInr7mbIcUFV9XVXdv1YL1n7d3HkOZClZq+pHs5ro4EFJvierxaB/ZN5U+7egn+tSci7m+a+qm0/LQ3y6qj5VVS+rqo2cNW4pWZeSc5+qukZV3aaqvrm+ui7sRlpK1mlG249kNendf05yflV917ypdlZVD6qqZ1TV06vqX86d50CWkrWqnp3VkjbnJXl/kkdW1W/Nm+rIc83ehqiqd3X37bZsH5PkvO6+9YyxdrSErFX1hCQ/l9XMppftG07yhSTP6+4nzJVtJ9Ob0l9I8sascn5Hkl/u7hfMGmwHC8v64SR37u6/n7avn9Ustxu3COxSfq5LyZks7vk/J8lvJfmDaej7kzy6u+84X6qdLSXrUnIml5eS/5JVMakkN8tqNtn/MWuwHSws64eS3Le7z5+2b5HVLK0btRj8VEq+Pl99rX5fVrMbP2q+VDtbWNYPJfnGfRNKVdXVsprl9BvnTXZkKXszW1IpWVLWfarqSZuYa7uFvSldUta3JbnbvtN2pk+g39zdd5432ZUt5ee6lJzJ4p7/93X3bbaNvbe7bztXpv1ZStal5EyWU0qSxWV9R3d/25btSvIXW8c2wZJKycKyvjLJo/bNHl1VN03yn3u17uJRwzV7M+vuJyV50hJKyZKybvHKqrp2d3+2qn44q0U/n7nTtPEz+/skl27ZvnQa20RLynp+krdX1cuyOl//AUneV1U/mSTd/Yw5w22zlJ/rUnImy3r+/0dVPT6rRao7q0/LX11V10uS7v7MnOG2WUrWpeRMkkv3lafJBbni/882yZKynltVr07y4qxeAw9O8o5aXceV7n7pnOG2OD/JTbJaUD1JTp7GNtGSsh6f5INV9RdZPf93yOo18fIk6e77zxnuSHFkb0NU1V2SvGcBpWRpWd+X5LZJbpPkt5M8P8n3dvd3zJlru6p6UVYL6V7hTel026g3pQvL+sQD7e/uXzpSWQ5mKT/XpeRMFvf8/80Bdnd3b8y1ZkvJupScSVJVz0ly01yxlPyvrBZb36RSsrSsLzzA7u7ujbiGt6rekuTbsrrG+PJSkuSSZLNKycKyHvC9Xne/5UhlmZOytyGWUkqSxWV9V3ffrqp+IcmF3X3W9msON8HC3pQuJuuSLOXnupScsCRLKSXJsrIuxZJKyZKysqLsbYillJJkcVnfkuQ1Sc5Mctckn0ry3u7+5lmDsVZV9Rvd/RNV9YqsPnm8gk365JG9t6Tnv6ru0d1v3Hda2XYbdpRkEVmXkpP1qaqf6e6nVNVvZuffAY+ZIRZHSFX9WXd/e1Vdmis+/5XVBxJfM1O0Wbhmb3NcOk2A8sNJ7jpd8Hr1mTPtz5Kyfl+SH0zy8O7+ZFXdJMlTZ850uYW9KV1M1iT7lgJ52qwpdmEpP9el5Jws5vnP6kOoNybZacKATrJJxWQpWZeSc1GlZElZk3xw+nrurCkOYkmlZElZk9w9Sbr7+LmDbAJlb3NsdCnZZjFZe7WA8jO2bP+vJC+aL9GVLOlN6ZKyPjrJwxZyOslSfq5LyZks6/nfd63jmXMH2YWlZF1KzmQhpWSypKx3SvKK7j577iAHsaRSsqSsb89qPgniNE4GtZRPoKrqt7v7YXPn2I2FZd3I04p3spSf61JyJot7/mXdY0vJmSRV9Z+6++fmzrEbC8u6iNfAUnImi8v67u7+1rlzbApH9ma2lFKSLCtrd3/79HXTP4G6zcEfsjGWlPVaVfWtWb02r6S733WE8xzIUn6uS8mZLOv55+h276zWr12CJWU9pqqum/3/DtiUZTd2zLehlpT1pH1L7Oxkk2aNPhKUvZktqJQsKmuSVNUxWS30uXELvW6xpDelS8p6oyRPz85ZO8k9jmycA1rKz3UpOZNlPf+3mmY43m7fh2ibVLKXknUpOZPllJJkWVlvleSd2f/vgE1ZdmNJpWRJWY9J8k+zrIK6NsreBlhIKUmyrKzd/eWq+nBV3WS6Vm8TLelN6ZKynt/dm5TnQJbyc11KzmRZz//fZOeJRDbRUrIuJWeynFKSLCvrXy7kNL4llZIlZf1Ed//y3CE2hbK3ARZSSpIsK+vkukk+UFV/keSz+wY3aObAJb0pXVLWJVnKz3UpOZfmC939sblD7NJSsi4lZ7KcUpIsK+tSLKmULCnrEgrpEaPsbY5NLyVbLSnrz88dgFn87NwBmNWSnv//OXeAq2ApWZeSk/V55twBdmlJpWRJWe85d4BNYjbODVFV37HT+CZOHb6ErFX1E0neluRd3f2lufPsT1Xdq7tfN3eO3VhS1iVZys91KTlhSarqYd3923Pn2I0lZV2Kqrrehl3ruF9LysoVKXszW0opSRaX9WlJ7pzVNQbnZfVJ79uSvM0vKwAAjgbK3syWVEqWlHWfqrpGktOyyn2n6XZxd9961mAcUVX1NVnNwnfp3Fk4cqrqmt39uW1jJ3b3p+fKBMCRUVWPTvK73f0Pc2eZ09XmDnC06+6f7u47J/m6JE9I8pkkZyZ5f1X95azhtllS1i2OS/I1Sa4z3f53krfPmugAquprqmopS1tsfNaq+raqOi/J+7J6nb63qm4/d66dVNU1dxg7cY4sB7KUnJN3VNXp+zaq6l9l9QHVxqmqa1XVz1fV/zdtn1pV9507106WknUpOVmfqnrWDrdfqaoHzJ1tq6p69LSsxcZbUtYkN8jq78CLq+reVbWk6w73jAlaNsdOpeS8WRPt38ZnrarnJfmmJJdmVe7eluQZm/rpTlV9W5IXJDl+tVkXJ/mR7n7nvMmubElZk5yV5Me7+61JUlXfnuSF2cwFwt9RVf+6u89JLi8mT0pyy3ljXclScibJDyZ5QVW9Ocn/k+T62awlIrZ6YVbT2t9p2r4wyR8leeVsifZvKVmXkjNV9awdhi9Jcm53v+xI5zmQJWVNcs2szkb6o2n7X2W1NMdtq+ru3f0TsyW7on2l5F1Z/X19bW/uqXeLydrd/6Gqfj7JvbI6OPGfq+rFSc7q7o/Mm+7IcRrnzHYoJeckOWcTS8nCsr4myYlJ3p9V0fvzJO/f1F9I0wLAj9pWSp69YYv/Jllc1ndvnyq8qt7V3bebK9P+VNU3Z/WH8835ajH50e7++Jy5tltKzn2q6oFJfier31t37e7zZ460o6o6t7tP2/qarar3dvdt58623VKyLiVncvnf151KyfWTXLBBpWRpWc9Jcpfu/vK0fWyStyb59iTnbdIlHdNRp32l5LQkG1tKlpQ1SarqtlllvXeSNyU5Pcnru/tnZg12hDiyN7+bJPknSf46q08dP57k4lkT7d9isnb3vsP135TV9Xo/leSfVdVnkvx5dz9x1oBX9uV95SlJuvvPqmpTJ8HZ+KxVta/MvaWqnpvkD7Ja9Pf7siopG6e7z6uqX80Vi8nGFail5EySqjoryS2yOpJ7yySvrKrf7O7fmjfZjr5QVcdl9TpNVd0iyefnjbRfS8m6lJzJ6jW6tZQ8J1tKyZzBdrCkrNfNaiHwS6btaye53rRm8Ea9Frq7q+qTST6Z5EtZZX9JVW1cKVlK1qp6bJKHJvl0kucneVx3f7GqrpbVe9mNybpOyt7MllRKlpQ1Wf0yyuo6rYuz+kV/SZL7JrlDko3IuqRSsqSsSZ6+bXvr872pR3cXUUyWknNyXlZHHTvJ31TVHZM8Y+ZM+/PEJK9JcnJV/V6SuyR52KyJ9m8pWZeSM1lQKcmysj4lyXumU7kryV2T/KequnaSP5kz2FZLKiVLyprkekke1N0f2zrY3V85mq7fdRrnBqmqG2f1x+jOWZWS63f3CfOm2tmmZ62qx2SV7c5Jvphp1tDpdl53f2XGeJerqjcdYHd398ZcX7SkrEtUq6VNnrnvVOOquk5W15k+fN5kV7SUnPtMR3Zu0t0fnjvLwVTV9bM6vaiyOkV+Y2cNXUrWBeV8eJL/kNUHZ5eXkqw+VPvF7n7cfOmuaElZk6SqbpjVh7xJ8o7u/t9z5tlJVf1SkhdsLyXTvm/s7g/OEGtHC8v68O4+a9vYk7v78XNlmoOyN7OllJJkcVmfkWlpiO7+xNx5mEdVnZDVJ5CnZMuZDN39mLkyHchSismCct4vydOSXKO7b1ZV35Lkl7v7/jNHu9yWI+Y76u53HaksB7OUrEvJud0SSsk+C8t6oyQ3zRX/BvzpfImubEmlZGFZX53k97r796bt30pyzU39cHJdnMY5v1Oyusj53y2glJyShWTt7p+cO8NVsaRSsqSsSV6d1URC5yXZmA8jdrK1mCTZyGKSLCfn5BezekP65iTp7vdU1c3nDLSD7accb9XZrNlDl5J1KTm3u1qSi7L6vfr1VfX1m1ZKtlhE1qr6tawuNfhAvvo3oJNsWtZ/VVWf215KZs60P4vKmuTlVfWVrCZnufhoK3qJI3uwEarqbdmhlHT32bOF2o+FZd3ImTd3UlXvzOpN6Ju3zBz4/u7+Z/Mmu6Kl5ExWM/F19+l1xdkY39cbOHMsR7f9lZJN/BBlYVk/nOQ23b1p1xJewXS2xMuzmul4Xyl57LypdraErFV1vS2bxyf571md7fULSdLdn5kj11wc2YPNcM0FHY1cUtbfqap/ndW6Wpf/sd/QX/Rf7O5L6oprvm7i0cil5EySD1TVDyY5pqpOTfKYbNii6lX1oAPt7+6XHqksB7OUrEvJuc0Dk3zDppeSyZKyXpDk6tnQWVi3lZIfzVdLyS9V1fU26W/VkrJmtb5mZ3VN6b6v95lunWTTzvBYK2UPNsOSSsmSsn4hyVOT/Pt8dRbOTf1Fv/HFZLKUnEny6Kye+89nNXnEa5P8yqyJrux+09evzep66DdO23fP6ue6ScVkKVmXknOrjS4l2ywp62VZzcb5hlzx79WmXHawpFKymKzdfbO5M2wSp3HCBqiqRyX51azWLby8lHT3xvzy3GdhWS9IcodNnYFvq6q6VlbF5F5Z/RF9bZJf6e7PzRpsm6XkXJqqel2SM/ZdDz1NgPHb3f0v5k12ZUvJupScSVJV/y3JbZNsaim53MKynrHT+CZedsB6VNWdc+U5Bl40W6AZKHuwARZWSpaU9XVJHtjdl82dhSOnql6RA6ynuKHXFn2wu79xy/bVknxg69imWErWpeRMllVKlpR1SZZUSpaStap+J6t1Yd+T5MvTcG/iBxPr5DRO2AznZ3W6yRIsKetnszqF503Z0E+gl1JMlpJz8rTp64OSfF2S3522fyDJ382S6ODeUFWv/f/bu5tQKcswjOP/WxcZpguLQIIsKQjzK3UTGBLSotwEkWCuWhUUiQUtWoTWrjIqgyAqqBb2sRCDRGjRd1FoZZ6yNpkUuCqpaBGpd4v3Hc+c47zjaeE8z3Xm+oE4Hwp/jnLO3PM+8zw0y02h2QCjmkOfp1FpVemUGpQUWiPirczcHBFHGPB9q7ZNmrqGEkBmgKLCVmAdsCzH/MqWr+yZVSAi9gLXA9UOJT1irdW/Ax0RG9qbAweTzNxeJGwalc5+EXEwM9ed77FatBuL3NTe/Sgz95bsGUaltfZOpaFErHVxZp6IiCWDns8BB4KXFBFHERlKxFrfBh6o/biwC83DnlkFFIaSHqVWJSqDiUonnH1Rsikzf2rvXw3sr3EZn40npaFEqbVLu5R3S++MuFooDSVire8Dq4EvmfrmdE0rUS44L+M0q4DSoKTUGhHHGPwOdHWbyQDzI2LptMFkfuGmQVQ6AbYDH7SfMw1gCXBP2aSpIuIvhi+PXTjCnKFUWlU6AXovmKcPSr2hBKhmgFJqjYiFwH3AFTRnwr0H3A88BBwGqhr2gMuA7yNCYShRat1ROqAGHvbMKqA0lCi10qzX75kH3Aks6vizpVU/mLRUOsnMA+3xENe1D/1Q29lgmbkAICIeB04Ar9N8XbcCiwumnUOlVaUTtIYSpVaaf/OTwOc0Z8I9QvN/4PbM/KZkWIcdpQP+hx2lA2YqMz8s3VADL+M0q0BEXNp39+xQkpmPFkrqpNQ6SEQcysy1pTsGiYiLqHgw6VHpBKld4w5n5qrzPVYDlVaFzojYx+RQspHmbMAAttU2lIi1HsnMFe3tuTRD/5U+ImY8RMQnmbl+wFX+oNmNs5qr+6PgK3tmFcjM36Y99ExEHAKqG6CUWiNiTd/dOTRX+mr+vreWycFkVURUOZgg0im2a9zfEbEVeIOmcQvNbrI1UmlV6FzaN5S8RN1DiVLrv70bmXk6In6tsVNpKFFqzcz17e8LSrfUoOYXPWZjQ2koUWoFdvXdPgX8DGwukzKcymCi0tlS2nb7LuDZ9lcCn7aP1UilVaFTYihpKbWuiog/29sBXNzer2owURpKlFojYh5wL3AN8C3wSmaeKltVjpdxmlWg3TGqpzeUPJWZP5Yp6qbUqkRlO2uVTtDaNc7GU0ScZvJqYwAX05xjWtVQAlqtKpSGErHWN2nenPgYuBU4npnbylaVU+u78WZjJTNvLt0wU0qt7WfL7uDcz2w9VqppiAma8+tqH0xUOkFg17iIeDgzn4iI3Qze+Kia8ytVWlU6ATJzbumGmVJqFfIqk0PJbTRn2NY6lCi1LutbcvwyzdELY8vDnlkFlIYSpVZgH/AHcIi+F/uVqn4waal0gsaucUfb3w8WrZgZlVaVTjOloUSptX/J8amIKNlSnJdxmlUgIg4wOZT0PgdFZu7q/EuFiLVOZOby0h0zEREbBj1e29bRKp1mZrWLiK8yc03X/ZqItXrJcR8Pe2YVEBtKlFpfBHZn5pHSLTY6Qw7Vru4HfUS8M+z5mq6YqrSqdJopDSVKrTaVl3Ga1eGziFghMpRU3xoRE8AZmu9xd7cHgP/D5A+llSX7+qkMJiqdoLFbXJ8bgV+APcAXNF/PWqm0qnTamFP6HKRSq03lK3tmBU0bSq4Fah5KlFpPAqu7ns/M4yPMMevUHvh8C80ZcCuBd4E9mfld0bABVFpVOs3MRsHDnllBSkOJWGu1nyUw69JufrQFeBLYmZnPF07qpNKq0mlmdqF4GadZWcdqGpLOQ6n18oh4sOvJzHx6lDFmw7QDySaaoeQq4Dlgb8mmLiqtKp1mZheahz2zspSGEqXWucAl+LM6VrmIeA1YDuynufI0UTipk0qrSqeZ2Sh4GadZQRFxAniBjqEkM3eOtqibWKuXcZqEiDjD5A53/T+Qq9v4RqVVpdPMbBQ87JkVpDSUiLV+nZk3lO4wMzMzK2lO6QCzMae0zFCpdWPpADMzM7PSfGXPrKCIWJSZv5fumAmlVjMzMzPzsGdmZmZmZjYreRmnmZmZmZnZLORhz8zMzMzMbBbysGdmZmZmZjYLedgzMzMzMzObhTzsmZmZmZmZzUL/AYkzcd1vvy9NAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1080x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "error = labels_tip - preds\n",
    "sq_error = error * error\n",
    "avg_error = np.mean(sq_error, axis=0)\n",
    "plt.figure(figsize=(15, 10))\n",
    "plt.xticks(rotation=90)\n",
    "plt.ylabel('Prediction Error (mm)')\n",
    "bar = plt.bar(df.columns[8:][[0,1,2,12,13,14,24,25,26,36,37,38,48,49,50,60,61,62]], avg_error)\n",
    "for i in range(0,18,3):\n",
    "    bar[i].set_color('coral')\n",
    "    bar[i+1].set_color('olivedrab')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tips only with FC Architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "lstm (LSTM)                  (None, 24, 64)            18688     \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 24, 64)            0         \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 64)                33024     \n",
      "_________________________________________________________________\n",
      "batch_normalization_v1_8 (Ba (None, 64)                256       \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 100)               6500      \n",
      "_________________________________________________________________\n",
      "activation (Activation)      (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_v1_9 (Ba (None, 100)               400       \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 64)                6464      \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 18)                1170      \n",
      "=================================================================\n",
      "Total params: 66,502\n",
      "Trainable params: 66,174\n",
      "Non-trainable params: 328\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'labels_tip' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-68-6944a26e5f42>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0mmodel_fc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Adam'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'mse'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m \u001b[0mhistory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_fc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels_tip\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mseq_length\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m9\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_split\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'labels_tip' is not defined"
     ]
    }
   ],
   "source": [
    "model_fc = tf.keras.models.Sequential()\n",
    "model_fc.add(LSTM(64, return_sequences=True, input_shape=(seq_length, 8)))\n",
    "model_fc.add(Dropout(0.2))\n",
    "model_fc.add(LSTM(64))\n",
    "model_fc.add(BatchNormalization())\n",
    "model_fc.add(Dense(100, input_dim=64))\n",
    "model_fc.add(Activation('relu'))\n",
    "model_fc.add(BatchNormalization())\n",
    "model_fc.add(Dropout(0.2))\n",
    "model_fc.add(Dense(64, input_dim=64))\n",
    "model_fc.add(Activation('relu'))\n",
    "model_fc.add(Dropout(0.2))\n",
    "model_fc.add(Dense(18, input_dim=64))\n",
    "\n",
    "model_fc.summary()\n",
    "model_fc.compile(optimizer='Adam', loss='mse')\n",
    "\n",
    "history = model_fc.fit(features, labels_tip, batch_size=seq_length, epochs=9, verbose=1, validation_split=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visual Model Error\n",
    "A similar visualizion is now done on this new more restricted model. These errors show 20% - 25% improvement on fingertip y position."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = model_fc.predict(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3sAAAJyCAYAAAB9tdmfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzde5hlZ1km/PshCRAwECAtxiQQxCDicG7DyQ85fConCTpyUhEiTkAB8VORxG8UdYYBUUBRYYgECCpgRBzCQQQ5MxigCZBw1ABBkgEThGAABRKe+WOvDpVOdXd1d+1ae638fte1r9rrXXtXbmoX1fvea633re4OAAAA83KNsQMAAACw+ZQ9AACAGVL2AAAAZkjZAwAAmCFlDwAAYIYOHjvAgTjiiCP62GOPHTsGAADAKN73vvd9vru3rbdv0mXv2GOPzY4dO8aOAQAAMIqq+vTu9jmNEwAAYIaUPQAAgBlS9gAAAGZI2QMAAJghZQ8AAGCGlD0AAIAZUvYAAABmSNkDAACYIWUPAABghpQ9AACAGVL2AAAAZkjZAwAAmCFlDwAAYIaUPQAAgBlaetmrqoOq6v1V9Zph+2ZV9e6qOq+q/rKqrjmMX2vYPm/Yf+yyswEAAMzVVhzZe2KSj67Z/t0kz+7u707yxSSPHsYfneSLw/izh8cBAACwH5Za9qrq6CT3T/KCYbuS3CvJK4aHnJ7kQcP9E4btDPvvPTweAACAfbTsI3t/kOTXknxz2L5Rkku6+7Jh+4IkRw33j0rymSQZ9n9pePyVVNVJVbWjqnZcfPHFy8wOAAAwWUsre1X1gCQXdff7NvP7dvep3b29u7dv27ZtM781AADAbBy8xO99tyQPrKr7Jbl2kusl+cMkh1fVwcPRu6OTXDg8/sIkxyS5oKoOTnL9JP+6xHwAAACztbQje919Sncf3d3HJnlYkjd3908leUuSnxge9sgkrxrunzlsZ9j/5u7uZeUDAACYszHW2Xtykl+uqvOyuCbvtGH8tCQ3GsZ/OcnJI2QDAACYhWWexnmF7n5rkrcO9z+Z5Ph1HvMfSR68FXkAAADmbowjewAAACzZlhzZAwAAVscpz79zLv3q58eOkcOuc0Se9pizxo4xW47sAQDA1cwqFL1kdXLMlbIHAAAwQ8oeAADADCl7AAAAM2SCFoCrqVW5OD9xgT4ALIMjewBXU6tS9JLVygIAc6HsAQAAzJCyBwAAMEPKHgAAwAwpewAAADOk7AEAAMyQsgcAADBDyh4AAMAMKXsAAAAzpOwBAADMkLIHAAAwQ8oeAADADCl7AAAAM6TsAQAAzJCyBwAAMEPKHgAAwAwpewAAADOk7AEAAMyQsgcAADBDyh4AAMAMKXsAAAAzpOwBAADMkLIHAAAwQ8oeAADADCl7AAAAM6TsAQAAzJCyBwAAMEPKHgAAwAwpewAAADOk7AEAAMyQsgcAADBDyh4AAMAMKXsAAAAzpOwBAADMkLIHAAAwQ8oeAADADCl7AAAAM6TsAQAAzJCyBwAAMEPKHgAAwAwpewAAADOk7AEAAMyQsgcAADBDyh4AAMAMKXsAAAAzpOwBAADMkLIHAAAwQ8oeAADADCl7AAAAM6TsAQAAzNDSyl5VXbuq3lNVH6yqD1fVbw/jL66qT1XVB4bb7YbxqqrnVNV5VXVOVd1hWdkAAADm7uAlfu+vJblXd3+5qg5J8s6q+tth35O6+xW7PP6+SY4bbndK8rzhKwAAAPtoaUf2euHLw+Yhw6338JQTkrxkeN5ZSQ6vqiOXlQ8AAGDOlnrNXlUdVFUfSHJRkjd297uHXU8dTtV8dlVdaxg7Ksln1jz9gmFs1+95UlXtqKodF1988TLjAwAATNZSy153X97dt0tydJLjq+o/JTklyS2TfH+SGyZ58j5+z1O7e3t3b9+2bdumZwYAAJiDLZmNs7svSfKWJPfp7s8Op2p+LcmLkhw/POzCJMesedrRwxgAAAD7aJmzcW6rqsOH+4cm+aEkH9t5HV5VVZIHJfnQ8JQzk/zMMCvnnZN8qbs/u6x8AAAAc7bM2TiPTHJ6VR2URak8o7tfU1VvrqptSSrJB5I8dnj865LcL8l5Sb6a5MQlZgMAAJi1pZW97j4nye3XGb/Xbh7fSR63rDwAAABXJ1tyzR4AAABbS9kDAACYIWUPAABghpQ9AACAGVL2AAAAZkjZAwAAmCFlDwAAYIaUPQAAgBlS9gAAAGZI2QMAAJghZQ8AAGCGlD0AAIAZUvYAAABm6OCxAwDjOeX5d86lX/382DFy2HWOyNMec9bYMQAAZsWRPbgaW4Wil6xODgCAOVH2AAAAZkjZAwAAmCFlDwAAYIaUPQAAgBlS9gAAAGZI2QMAAJghZQ8AAGCGLKoOAFdTpzz/ziuxzuVh1zkiT3vMWWPHAJgdR/YA4GpqFYpesjo5AOZG2QMAAJghZQ8AAGCGlD0AAIAZUvYAAABmSNkDAACYIWUPAABghpQ9AACAGVL2AAAAZkjZAwAAmCFlDwAAYIaUPQAAgBlS9gAAAGZI2QMAAJghZQ8AAGCGlD0AAIAZUvYAAABmSNkDAACYIWUPAABghpQ9AACAGVL2AAAAZkjZAwAAmCFlDwAAYIaUPQAAgBlS9gAAAGZI2QMAAJghZQ8AAGCGDh47AAAAW++U5985l37182PHyGHXOSJPe8xZY8eAWXJkDwDgamgVil6yOjlgjpQ9AACAGVL2AAAAZkjZAwAAmCFlDwAAYIaWVvaq6tpV9Z6q+mBVfbiqfnsYv1lVvbuqzquqv6yqaw7j1xq2zxv2H7usbAAAAHO3zCN7X0tyr+6+bZLbJblPVd05ye8meXZ3f3eSLyZ59PD4Ryf54jD+7OFxAAAA7Iellb1e+PKwechw6yT3SvKKYfz0JA8a7p8wbGfYf++qqmXlAwAAmLOlXrNXVQdV1QeSXJTkjUk+keSS7r5seMgFSY4a7h+V5DNJMuz/UpIbrfM9T6qqHVW14+KLL15mfAAAgMlaatnr7su7+3ZJjk5yfJJbbsL3PLW7t3f39m3bth1wRgAAgDnaktk4u/uSJG9Jcpckh1fVwcOuo5NcONy/MMkxSTLsv36Sf92KfAAAAHOzzNk4t1XV4cP9Q5P8UJKPZlH6fmJ42COTvGq4f+awnWH/m7u7l5UPAABgzg7e+0P225FJTq+qg7IolWd092uq6iNJXl5V/z3J+5OcNjz+tCR/VlXnJflCkoctMRsAAMCsLa3sdfc5SW6/zvgns7h+b9fx/0jy4GXlAQAAuDrZkmv2AAAA2FrKHgAAwAwpewAAADOk7AEAAMyQsgcAADBDyh4AAMAMKXsAAAAzpOwBAADMkLIHAAAwQ8oeAADADCl7AAAAM6TsAQAAzNDBYweA2fm9E5OvXDJ2ioXrHp486UVjpwAAYASO7MFmW5Wil6xWFgAAtpSyBwAAMEPKHgAAwAwpewAAADOk7AEAAMyQsgcAADBDyh4AAMAMKXsAAAAzpOwBAADMkLIHAAAwQ8oeAADADCl7AAAAM6TsAQAAzJCyBwAAMEPKHgAAwAwpewAAADOk7AEAAMyQsgcAADBDyh4AAMAMKXsAAAAzpOwBAADMkLIHAAAwQ8oeAADADCl7AAAAM6TsAQAAzJCyBwAAMEPKHgAAwAwpewAAADOk7AEAAMyQsgcAADBDyh4AAMAMKXsAAAAzpOwBAADMkLIHAAAwQ8oeAADADCl7AAAAM6TsAQAAzJCyBwAAMEPKHgAAwAwpewAAADOk7AEAAMyQsgcAADBDyh4AAMAMKXsAAAAztLSyV1XHVNVbquojVfXhqnriMP5bVXVhVX1guN1vzXNOqarzqurjVfUjy8oGAAAwdwcv8XtfluRXuvvsqjosyfuq6o3Dvmd39++vfXBV3SrJw5J8X5LvTPL3VXWL7r58iRkBAABmaWlH9rr7s9199nD/0iQfTXLUHp5yQpKXd/fXuvtTSc5Lcvyy8gEAAMzZllyzV1XHJrl9kncPQ4+vqnOq6oVVdYNh7Kgkn1nztAuyTjmsqpOqakdV7bj44ouXmBoAAGC6ll72qurbkvx1kl/q7n9L8rwkN09yuySfTfLMffl+3X1qd2/v7u3btm3b9LwAAABzsNSyV1WHZFH0/qK7X5kk3f0v3X15d38zyZ/mW6dqXpjkmDVPP3oYAwAAYB8tczbOSnJako9297PWjB+55mE/luRDw/0zkzysqq5VVTdLclyS9ywrHwAAwJxteDbO4dq670zy70nOH47M7cndkjwiyblV9YFh7NeTPLyqbpekk5yf5DFJ0t0frqozknwki5k8H2cmTgAAgP2zx7JXVddP8rgkD09yzSQXJ7l2khtX1VlJntvdb1nvud39ziS1zq7X7e6/191PTfLUjUUHAABgd/Z2ZO8VSV6S5P/p7kvW7qiqOyZ5RFV9V3eftqyAAAAA7Ls9lr3u/qE97HtfkvdteiIAAAAO2L5cs3ebJMeufc7OGTYBAABYLRsqe1X1wiS3SfLhJDsnZukkyh4AAMAK2uiRvTt3962WmgQAAIBNs9F19v6hqpQ9AACAidjokb2XZFH4Ppfka1ksqdDdfZulJQMAAGC/bbTsnZZhgfR865o9AAAAVtRGy97F3X3mUpMAAACwaTZa9t5fVS9N8uosTuNMYukFAACAVbXRsndoFiXvh9eMWXoBAABgRW2o7HX3icsOAgAAwObZ6KLqN0vyhCTHrn1Odz9wObEAAAA4EBs9jfN/ZTEj56tjNk4AAICVt9Gy9x/d/ZylJgEAAGDTbLTs/WFVPSXJG3Ll2TjPXkoqAAAADshGy96ts1hU/V751mmcPWwDAACwYjZa9h6c5Lu6++vLDAMAAMDmuMYGH/ehJIcvMwgAAACbZ6NH9g5P8rGqem+ufM2epRcAAABW0EbL3lOWmgIAAIBNtceyV1XVC2/b22M2PxoAAAD7a2/X7L2lqp5QVTdZO1hV16yqe1XV6Ukeubx4AAAA7I+9ncZ5nyQ/m+RlVXWzJJckOTSLkviGJH/Q3e9fbkQAAAD21R7LXnf/R5LnJnluVR2S5Igk/97dl2xFOAAAAPbPRidoSXd/I8lnl5gFAACATbLRdfYAAACYEGUPAABghvZa9qrqoKp6y1aEAQAAYHPstex19+VJvllV19+CPAAAAGyCjU7Q8uUk51bVG5N8Zedgd//iUlIBAABwQDZa9l453AAAAJiADZW97j69qq6Z5BbD0MeHpRgAAABYQRsqe1V1jySnJzk/SSU5pqoe2d1vX140AAAA9tdGT+N8ZpIf7u6PJ0lV3SLJy5LccVnBAAAA2H8bXWfvkJ1FL0m6+x+THLKcSAAAAByojR7Z21FVL0jy58P2TyXZsZxIAAAAHKiNlr2fT/K4JDuXWnhHkucuJREAAAAHbK9lr6oOSvLC7v6pJM9afiQAAAAO1F6v2evuy5PcdFh6AQAAgAnY6Gmcn0zyv6vqzCRf2TnY3Y70AQAArKCNlr1PDLdrJDlseXEAAADYDBu9Zu+w7v7VLcgDAADAJtjoNXt324IsAAAAbJKNnsb5geF6vb/Kla/Ze+VSUgEAAHBANlr2rp3kX5Pca81YJ1H2AAAAVtCGyl53n7jsIAAAAGyePV6zV1VnrLn/u7vse8OyQgEAAHBg9jZBy3Fr7v/QLvu2bXIWAAAANsneyl7v5z4AAABGtLdr9q5TVbfPohQeOtyv4XbossMBAACwf/ZW9j6b5FnD/c+tub9zGwAAgBW0x7LX3ffcqiAAAABsnr1dswcAAMAEKXsAAAAzpOwBAADM0IbLXlUdVVV3raq777zt5fHHVNVbquojVfXhqnriMH7DqnpjVf3T8PUGw3hV1XOq6ryqOqeq7nBg/9MAAACuvvY2G2eSpKp+N8lDk3wkyeXDcCd5+x6edlmSX+nus6vqsCTvq6o3JnlUkjd199Or6uQkJyd5cpL7ZrGI+3FJ7pTkecNXAAAA9tGGyl6SByX5nu7+2ka/cXd/NoulG9Ldl1bVR5McleSEJPcYHnZ6krdmUfZOSPKS7u4kZ1XV4VV15PB9AAAA2AcbPY3zk0kO2d//SFUdm+T2Sd6d5MZrCtznktx4uH9Uks+sedoFwxgAAAD7aKNH9r6a5ANV9aYkVxzd6+5f3NsTq+rbkvx1kl/q7n+rqiv2dXdXVe9L4Ko6KclJSXKTm9xkX54KAABwtbHRsnfmcNsnVXVIFkXvL7r7lcPwv+w8PbOqjkxy0TB+YZJj1jz96GHsSrr71CSnJsn27dv3qSgCAABcXWyo7HX36VV1zSS3GIY+3t3f2NNzanEI77QkH+3uZ63ZdWaSRyZ5+vD1VWvGH19VL89iYpYvuV4PAABg/2x0Ns57ZDGZyvlJKskxVfXI7t7TbJx3S/KIJOdW1QeGsV/PouSdUVWPTvLpJA8Z9r0uyf2SnJfFaaMn7tP/EgAAAK6w0dM4n5nkh7v740lSVbdI8rIkd9zdE7r7nVkUw/Xce53Hd5LHbTAPAAAAe7DR2TgP2Vn0kqS7/zEHMDsnAAAAy7XRI3s7quoFSf582P6pJDuWEwkAAIADtdGy9/NZnGK5c6mFdyR57lISAQAAcMA2Ohvn15I8a7gBAACw4vZY9qrqjO5+SFWdm+Qqa9p1922WlgwAAID9trcje08cvj5g2UEAAADYPHucjXPNoua/0N2fXntL8gvLjwcAAMD+2OjSCz+0zth9NzMIAAAAm2dv1+z9fBZH8G5eVees2XVYknctMxgAAAD7b2/X7L00yd8meVqSk9eMX9rdX1haKgAAAA7I3q7Z+1J3n5/kD5N8Yc31epdV1Z22IiAAAAD7bqPX7D0vyZfXbH95GAMAAGAFbbTsVXdfsc5ed38zG1yQHQAAgK230bL3yar6xao6ZLg9McknlxkMAACA/bfRsvfYJHdNcmGSC5LcKclJywoFAADAgdnQqZjdfVGShy05CwAAAJtkb+vs/Vp3P6Oq/ihJ77q/u39xackAAADYb3s7svfR4euOZQcBAABg8+yx7HX3q4evp29NHAAAADbD3k7jfHXWOX1zp+5+4KYnAgAA4IDt7TTO3x++/niS70jy58P2w5P8y7JCAQAAcGD2dhrn25Kkqp7Z3dvX7Hp1VbmODwAAYEVtdJ2961bVd+3cqKqbJbnuciIBAABwoDa0zl6S/y/JW6vqk0kqyU2TPGZpqQAAADggG11U/fVVdVySWw5DH+vury0vFgAAAAdiQ6dxVtV1kjwpyeO7+4NJblJVD1hqMgAAAPbbRq/Ze1GSrye5y7B9YZL/vpREAAAAHLCNlr2bd/czknwjSbr7q1lcuwcAAMAK2mjZ+3pVHZphgfWqunkS1+wBAACsqI3OxvmUJK9PckxV/UWSuyV51LJCAQAAcGD2WvaqqpJ8LMmPJ7lzFqdvPrG7P7/kbAAAAOynvZa97u6qel133zrJa7cgEwAAAAdoo9fsnV1V37/UJAAAAGyajV6zd6ckP11V5yf5ShancnZ332ZZwQAAANh/Gy17P7LUFAAAAGyqPZa9qrp2kscm+e4k5yY5rbsv24pgAAAA7L+9XbN3epLtWRS9+yZ55tITAQAAcMD2dhrnrYZZOFNVpyV5z/IjAQAAcKD2dmTvGzvvOH0TAABgOvZ2ZO+2VfVvw/1KcuiwvXM2zustNR0AAAD7ZY9lr7sP2qogAAAAbJ6NLqoOAADAhCh7AAAAM6TsAQAAzJCyBwAAMEPKHgAAwAwpewAAADOk7AEAAMyQsgcAADBDyh4AAMAMKXsAAAAzpOwBAADMkLIHAAAwQ8oeAADADCl7AAAAM6TsAQAAzJCyBwAAMENLK3tV9cKquqiqPrRm7Leq6sKq+sBwu9+afadU1XlV9fGq+pFl5QIAALg6WOaRvRcnuc8648/u7tsNt9clSVXdKsnDknzf8JznVtVBS8wGAAAwa0sre9399iRf2ODDT0jy8u7+Wnd/Ksl5SY5fVjYAAIC5G+OavcdX1TnDaZ43GMaOSvKZNY+5YBi7iqo6qap2VNWOiy++eNlZAQAAJmmry97zktw8ye2SfDbJM/f1G3T3qd29vbu3b9u2bbPzAQAAzMKWlr3u/pfuvry7v5nkT/OtUzUvTHLMmocePYwBAACwH7a07FXVkWs2fyzJzpk6z0zysKq6VlXdLMlxSd6zldkAAADm5OBlfeOqelmSeyQ5oqouSPKUJPeoqtsl6STnJ3lMknT3h6vqjCQfSXJZksd19+XLygYAADB3Syt73f3wdYZP28Pjn5rkqcvKAwAAcHUyxmycAAAALJmyBwAAMEPKHgAAwAwpewAAADOk7AEAAMyQsgcAADBDyh4AAMAMKXsAAAAzpOwBAADMkLIHAAAwQ8oeAADADCl7AAAAM6TsAQAAzJCyBwAAMEPKHgAAwAwpewAAADOk7AEAAMyQsgcAADBDyh4AAMAMKXsAAAAzpOwBAADMkLIHAAAwQ8oeAADADB08dgCAjTj25NeOHSFJcv7T7z92BACADXFkDwAAYIaUPQAAgBlS9gAAAGZI2QMAAJghZQ8AAGCGlD0AAIAZUvYAAABmSNkDAACYIWUPAABghpQ9AACAGVL2AAAAZkjZAwAAmCFlDwAAYIaUPQAAgBlS9gAAAGZI2QMAAJghZQ8AAGCGlD0AAIAZUvYAAABmSNkDAACYIWUPAABghg4eOwAAAMzC752YfOWSsVMsXPfw5EkvGjsFI3NkDwAANsOqFL1ktbIwGmUPAABghpQ9AACAGVL2AAAAZkjZAwAAmCFlDwAAYIaUPQAAgBlS9gAAAGbIouoAAMDKOvbk144dIUly/tPvP3aEfebIHgAAwAwpewAAADO0tLJXVS+sqouq6kNrxm5YVW+sqn8avt5gGK+qek5VnVdV51TVHZaVCwAA4OpgmUf2XpzkPruMnZzkTd19XJI3DdtJct8kxw23k5I8b4m5AAAAZm9pZa+7357kC7sMn5Dk9OH+6UketGb8Jb1wVpLDq+rIZWUDAACYu62+Zu/G3f3Z4f7nktx4uH9Uks+sedwFw9hVVNVJVbWjqnZcfPHFy0sKAAAwYaNN0NLdnaT343mndvf27t6+bdu2JSQDAACYvq0ue/+y8/TM4etFw/iFSY5Z87ijhzEAAAD2w1Yvqn5mkkcmefrw9VVrxh9fVS9PcqckX1pzuifAtPzeiclXLhk7RXLdw5MnvWjsFADASJZW9qrqZUnukeSIqrogyVOyKHlnVNWjk3w6yUOGh78uyf2SnJfkq0lOXFYugKVbhaKXrE4OAGAUSyt73f3w3ey69zqP7SSPW1YWAACAq5utPo0TAPbZsSe/duwISZLzn37/sSMAwIaNNhsnAAAAy+PIHgDAZjFBE7BCHNkDANgsq1D0ktXJAYzKkT0AAFaa63Zh/yh7ALCZnMYHwIpwGicAbKZVKHrJ6uQAYDTKHgAAwAwpewAAADOk7AEAAMyQsgcAADBDyh4AAMAMKXsAAAAzpOwBAADMkLIHAAAwQ8oeAADADB08dgAAgD059uTXjh3hCuc//f5jRwDYMEf2AAAAZkjZAwAAmCFlDwAAYIaUPQAAgBlS9gAAAGZI2QMAAJghZQ8AAGCGlD0AAIAZUvYAAABmSNkDAACYIWUPAABghpQ9AACAGVL2AAAAZkjZAwAAmCFlDwAAYIaUPQAAgBlS9gAAAGZI2QMAAJghZQ8AAGCGlD0AAIAZUvYAAABmSNkDAACYIWUPAABghpQ9AACAGVL2AAAAZkjZAwAAmCFlDwAAYIaUPQAAgBlS9gAAAGZI2QMAAJghZQ8AAGCGlD0AAIAZUvYAAABmSNkDAACYIWUPAABghpQ9AACAGVL2AAAAZkjZAwAAmCFlDwAAYIYOHuM/WlXnJ7k0yeVJLuvu7VV1wyR/meTYJOcneUh3f3GMfFcnpzz/zrn0q58fO0YOu84Redpjzho7BgAAzMaYR/bu2d236+7tw/bJSd7U3ccledOwzZKtQtFLVicHAADMxSqdxnlCktOH+6cnedCIWQAAACZtrLLXSd5QVe+rqpOGsRt392eH+59LcuP1nlhVJ1XVjqracfHFF29FVgAAgMkZ5Zq9JD/Q3RdW1bcneWNVfWztzu7uqur1ntjdpyY5NUm2b9++7mMAAACu7kY5stfdFw5fL0ryN0mOT/IvVXVkkgxfLxojGwAAwBxsedmrqutW1WE77yf54SQfSnJmkkcOD3tkkldtdTYAAIC5GOM0zhsn+Zuq2vnff2l3v76q3pvkjKp6dJJPJ3nICNkAAABmYcvLXnd/Mslt1xn/1yT33uo8AAAAc7RKSy8AAACwSZQ9AACAGVL2AAAAZkjZAwAAmCFlDwAAYIaUPQAAgBlS9gAAAGZI2QMAAJghZQ8AAGCGlD0AAIAZUvYAAABmSNkDAACYIWUPAABghpQ9AACAGVL2AAAAZkjZAwAAmCFlDwAAYIaUPQAAgBlS9gAAAGZI2QMAAJghZQ8AAGCGlD0AAIAZUvYAAABmSNkDAACYIWUPAABghpQ9AACAGVL2AAAAZkjZAwAAmCFlDwAAYIaUPQAAgBlS9gAAAGZI2QMAAJghZQ8AAGCGlD0AAIAZUvYAAABmSNkDAACYIWUPAABghpQ9AACAGTp47ACz9HsnJl+5ZOwUyXUPT570orFTAAAAI3BkbxlWoeglq5MDAADYcsoeAADADCl7AAAAM6TsAQAAzJCyBwAAMEPKHgAAwAwpewAAADOk7AEAAMyQsgcAADBDyh4AAMAMKXsAAAAzpOwBAADM0MFjB4CNOvbk144dIUly/tPvP3YEAADYK0f2AAAAZkjZAwAAmCFlDwAAYIaUPQAAgBlS9gAAAGZo5cpeVd2nqj5eVedV1clj5wEAAJiilSp7VXVQkj9Jct8kt0ry8Kq61bipAAAApmelyl6S45Oc192f7O6vJ3l5khNGzgQAADA51d1jZ7hCVf1Ekvt0988N249Icqfufvyax5yU5KRh83uSfHzLg26NI5J8fuwQGyTrckwl61RyJrIuy1SyTiVnIuuyTCXrVHImsi7LVLJOJWcyraz76qbdvW29HQdvdZID1d2nJjl17BzLVlU7unv72NUaJZoAABs7SURBVDk2QtblmErWqeRMZF2WqWSdSs5E1mWZStap5ExkXZapZJ1KzmRaWTfTqp3GeWGSY9ZsHz2MAQAAsA9Wrey9N8lxVXWzqrpmkoclOXPkTAAAAJOzUqdxdvdlVfX4JH+X5KAkL+zuD48cayxTOlVV1uWYStap5ExkXZapZJ1KzkTWZZlK1qnkTGRdlqlknUrOZFpZN81KTdACAADA5li10zgBAADYBMoeAADADCl7ALCChonKVk5V3WjsDABsjLK3Iqrq/11n7JFjZNmbqnpmVX3f2Dk2oqo+UVWP3WXsNWPl2Z2qutU6Y/cYIcpeVdUTquoGY+fYiKp6U1Xdb5exlbxA2+/A5pvY6//Wqjp2zfbxWcxQvYrOqqq/qqr7VVWNHWZPquoRVXXYLmMPGCvP7lTVt68z9j1jZNmbqvrRqprE+8eqOq2qbrfL2G+NFGePqurRu2wfVFVPGSvP7lTVK6vq/lP4Haiqy6vq6Wv/TlXV2WNmGsPKv1BXI79ZVc+rqutW1Y2r6tVJfnTsULvx0SSnVtW7q+qxVXX9sQPtwTeS3LOqXrTmU/Kjxgy0G2dU1ZNr4dCq+qMkTxs71G7cOMl7q+qMqrrPir/Zu1mSJ+/yD+aqLqjqd2DzTen1f1qS11fVL1TVU5P8zyQnjpxpd26Rxax2j0jyT1X1P6rqFiNn2p0/SvKOqvreNWO/M1aYPXhHVT1k50ZV/UqSvxkxz548NIvX/RlVdcuxw+zFjyQ5vap+Zs3YA8cKsxf3rqrXVdWRwwfqZyU5bG9PGsFzk/xkFr8DT1/VDyUGH86i67yhqm44jK3qv1dLo+ytjh9M8okkH0jyziQv7e6fGDfS+rr7Bd19tyQ/k+TYJOdU1Uur6p7jJlvXV7v7oVkU1HdU1U2SrOIUtHdKckySd2Xxaf7/SXK3URPtRnf/1yTHJTktyaPyrTd7Nx812PouSXLvJDeuqlev+AcTfgc232Re/+7+uySPTfKHSX42yf26eyU/ge6FN3b3w5P8lySPTPKeqnpbVd1l5Hi7+lQWP89XVNWDh7FVfLN3jySPGI6Yvj2LQn38uJHW190/neT2WbxneXFV/UNVnbTrEdQVcVGSuyd5cFX9SVUdnNV8/dPdP5nk9CTnJnldkl/q7l8dN9VVdfffd/dPJblDkvOT/H1VvauqTqyqQ8ZNdxWXdfevJXlBFu8B75jVfA+4VMre6rhBFn/YP5Hka0luusKflqeqDkpyy+H2+SQfTPLLVfXyUYNdVSVJdz8jyf+f5A1Jjh410fq+keTfkxya5NpJPtXd3xw30u71Ys2Wzw23y7L4/X1FVT1j1GBXVd19WXf/QpK/zuKDlKucLrUi/A5svsm8/lX1G1kchbp7kt9K8taquv+ooXajqm5UVU+sqh1JfjXJE5IckeRXkrx01HBX1UNp/sEkJ1XV72exju9K6e7PJnl9krtk8SHq6d395VFD7UF3/1uSVyR5eZIjk/xYkrOr6gmjBruq6u4vdfePJrk4yVuTrOSHPlV1XJInZvG36tNZlP/rjJtqfbW4bvdRSX4uyfuz+JDqDkneOGKs9ex8D/iXWRyRflGS7xo10Ri6220Fbkn+McnPDvcPTfKcJO8aO9dusj47yT8leX6S43fZ9/Gx8+2S50d32b5pkt8cO9c6OT+YxalFh2TxD+erkvzV2Ll2k/WJSd6X5O+SPDjJIcP4NZJ8Yux8u2R9zC7bd0zywrFz+R3w+q+T9Q+SHLpm+6ZJ3jh2rt1k/cckv5Hk6HX2PXnsfLvkee2a+9dI8ntJvjl2rnVy/n2SlyQ5PMmtk7wnye+PnWs3WR+YxSmm5yZ5UpJvH8avk+T8sfPtkvW3d9n+0SRvHjvXbrJ+LMm9h/uVxYcnHx471zo5/ybJR5KckuTIXfbtGDvfLnnuuMv29ZP8zNi5tvpmUfUVUVU36e5/3mXs7t399rEy7U5VnZjkjO7+yjr7rt/dXxoh1qRV1fbu3rHL2CO6+8/GyrQ7VfXbWbxh/vQ6+763uz86QqzJ8zvAVFRVtTcPm6qqHtTd/2vN9sFJTunu/zZirHVV1elJTlvv/UlV3bu73zRCrMmrquv14ojp2rFbdPc/jpVpPVV1z+5+y9g52DhlDwAAYIZcswcAADBDB48dAJapqv6sux+xtzHmq6q+I4vJjzrJe7v7cyNHYgt5/eHqbVh26ZZZ/A34eHd/feRIbKGq+vEkP5DF6//O7l7VJU2WxpG9FVFVV7kuZ72xVTClrEmutPj7MIvoHUfKskdV9R1V9cBaLFj7HWPn2ZOpZK2qn8tiooMfT/ITWSwG/bPjptq9qfxck2lkndLrX1XfNSwP8fmquqiqXlVVKzlrnKzLUVXXrKrbVNWt61vrwq6kqWQdZrT9RBaT3v1xkvOq6r7jptq9qvrxqnpWVT2zqn5s7Dy7M6Gcz81iSZtzk3woyWOq6k/GTbX1XLO3Iqrq7O6+w5rtg5Kc2923GjHWuqaQtapOSfLrWcxs+tWdw0m+nuTU7j5lrGzrGd6U/maSN2eR8weT/E53v3DUYOuYWNaPJ7lrd//rsH2jLGa5XblFYCf2c51E1om9/mcl+ZMkLxuGHpbkCd19p/FSrU/WzTeUkv+ZRTGpJDfLYjbZvx012DomlvVjSR7Q3ecN2zfPYobWlVsMfigm351v/a4+NIvZjR83XqqrmkrO5IrX/3t3TihVVdfIYobT7x032dZS9kY2pVIypaw7VdXTVjHXrib2pnRKWd+V5B47T9sZPoF+a3ffddxkVzWxn+sksk7s9T+nu2+zy9gHu/u2Y2XaHVk338RKyZSyvre7v3/NdiV5z9qxVTGVYjKVnElSVa9J8ridM0dX1U2T/HEv1l282nDN3si6+2lJnjaFUjKlrGu8pqqu291fqaqfzmLRzz9cb8r4kf1rkkvXbF86jK2iKWU9L8m7q+pVWZyvf0KSc6rql5Oku581ZrhdTOnnOpWsU3r9/7aqTs5ikerO4tPy11XVDZOku78wZrhdyLr5Lt1ZngafzJX/P7ZKppR1R1W9LskZWbz+D07y3lpcx5XufuWY4XZxXpKbZLGgepIcM4ytmqnkTJLDkny0qt6Txet/fBa/E2cmSXc/cMxwW8WRvRVRVXdL8oEJlJKpZT0nyW2T3CbJi5O8IMlDuvsHx8y1q6p6SRYL6V7pTelwW6k3pRPL+pQ97e/u396qLHszsZ/rJLJO7PX/1B52d3evzHVmsm6+qnpekpvmyqXkn7NYbH2lSsnEsr5oD7u7u1fmGt6qeluS78/iOuMrikmSLyWrU0ymkjNJqmqP7/W6+21blWVMyt6KmEopSSaX9ezuvkNV/WaSC7v7tF2vOVwFE3tTOpmsUzKln+uUssIUTKyUTCbrlEylmEwlJ9+i7K2IqZSSZHJZ35bk9UlOTHL3JBcl+WB333rUYCxVVf1Bd/9SVb06i08er2SVPnlk803p9a+qe3X3m3eeVrarFTtKIiuTUFW/1t3PqKo/yvp/A35xhFhskap6Z3f/QFVdmiu//pXFBxLXGynaKFyztzouHSZA+ekkdx8ueD1k5Ey7M6WsD03yk0ke3d2fq6qbJPm9kTNdYWJvSieTNcnOpUB+f9QUGzCln+uEsk7m9c/iQ6g3J1lvwoBOskqlRNZNNqVSMqWsST46fN0xaooNmEoxmUrOwT2TpLsPGzvIKlD2VsdKl5JdTCZrLxZQftaa7X9O8pLxEl3FlN6UTinrE5I8aiKnk0zp5zqVrFN6/Xde53ji2EE2QNbNN5lSkmllvUuSV3f36WMH2YCpFJOp5EySd2cxnwRxGiczNZVPoKrqxd39qLFzbMTEsq7kacXrmdjPdRJZJ/b6y7oEU8laVf+ju3997BwbMbGsk3j9k+lknUrOJKmq93f37cfOsSoc2RvZVEpJMq2s3f0Dw9dV/wTqNnt/yMqYUtbrVNXts/jdvIruPnuL8+zJlH6uU8k6pdefq7f7ZLF+7RRMKetBVXWD7P5vwKosuZHsJuMKmkrOJNm2c4md9azKjNFbRdkb2YRKyaSyJklVHZTFQp8rt9DrGlN6UzqlrEcleWbWz9pJ7rW1cfZoSj/XqWSd0ut/y2GG413t/BBtlQq2rJtvSqVkSllvmeR92f3fgJVYcmMwlWIylZxJclCSb8u0CurSKHsrYCKlJMm0snb35VX18aq6yXCt3iqa0pvSKWU9r7tXKc+eTOnnOpWsU3r9P5X1JxFZRbJuvimVkill/ciETuObSjGZSs4k+Wx3/87YIVaFsrcCJlJKkkwr6+AGST5cVe9J8pWdgys0a+CU3pROKeuUTOnnOqWsU/H17v702CE2SNbNN6VSMqWsUzKVYjKVnMk0CumWUfZWx6qXkrWmlPU3xg7AKJ48dgBGNaXX/3+PHWAfyMpU/OHYAfbBVIrJVHImyb3HDrBKzMa5IqrqB9cbX8Wpw6eQtap+Kcm7kpzd3ZeNnWd3quqHu/sNY+fYiCllnZIp/VynlBWmoKoe1d0vHjvHRkwp65RU1Q1X7HrHdU0lJ1el7I1sKqUkmVzW309y1yyuMTg3i09535XkXf5YAQBwdaDsjWxKpWRKWXeqqmsm2Z5F7rsMt0u6+1ajBmNLVdX1spiB79Kxs7B1qura3f0fu4wd0d2fHysTAFujqp6Q5M+7+4tjZxnTNcYOcHXX3b/a3XdN8h1JTknyhSQnJvlQVX1k1HC7mFLWNQ5Ncr0k1x9u/yfJu0dNtAdVdb2qmsrSFiuftaq+v6rOTXJOFr+nH6yqO46daz1Vde11xo4YI8veTCjre6vqzjs3quo/Z/EB1cqpqutU1W9U1Z8O28dV1QPGzrUeWZmKqnrOOrf/VlUnjJ1tV1X1hGFpi5U2lZyDG2fx78AZVXWfqprSdYebxgQtq2O9UnLuqIl2b+WzVtWpSb4vyaVZlLt3JXnWqn66U1Xfn+SFSQ5bbNYlSX62u983brKrmlLWJKcl+YXufkeSVNUPJHlRVnNx8PdW1X/p7rOSK4rJ05LcYtxY65pK1p9M8sKqemuS70xyo6zO8hC7elEW09rfZdi+MMlfJXnNaIl2T9ZNVlXPWWf4S0l2dPertjrPnkwpa5JrZ3E20l8N2/85i2U5bltV9+zuXxot2VXtLCZnZ/Fv7N/1ap5+N5Wc6e7/WlW/keSHszg48cdVdUaS07r7E+Om2zpO4xzZOqXkrCRnrWIpmVjW1yc5IsmHsih6/5DkQ6v6B2lY/Pdxu5SS567Qwr9XmFjW9+86VXhVnd3ddxgr0+5U1a2z+IfzrflWMfm57r5gzFzrmVjWByX5syz+bt29u88bOdK6qmpHd29f+ztbVR/s7tuOnW1Xsm6+4d/X9UrJjZJ8cpVKycSynpXkbt19+bB9cJJ3JPmBJOeu2iUdw5GnncVke5KVLCZTyblTVd02i6z3SfKWJHdO8sbu/rVRg20RR/bGd5Mk10ryT1l84nhBkktGTbR7k8na3TsP139fFtfr/UqS/1RVX0jyD939lFEDXtXlO8tTknT3O6tqVSfBWfmsVbWzzL2tqp6f5GVZLPr70CwKysrp7nOr6qm5cjFZufKUTCdrVZ2W5OZZHMm9RZLXVNUfdfefjJtsXV+vqkOz+D1NVd08ydfGjbRbsm6+2+TKpeR5WVNKxgy2jillvUEWC4F/adi+bpIbDmsGr9zvQXd3VX0uyeeSXJZF/ldU1UoVk6nkrKonJvmZJJ9P8oIkT+rub1TVNbJ4L7syWZdJ2RvZlErJlLImiz9GWVyndUkWf+i/lOQBSY5PshJZp1RKppQ1yTN32V77eq/q0d3JFJMJZT03iyOOneT/tnf/oXbXdRzHn6+NamaazJCGpVOUstSZSj/MEpGKGsZQEqZ/WFgUWBsaRVTYphTlLzKLIFJSG1pRptFQyzQr++Vsuq0lhCZFIyLNQtHUvfvj8z27597z497i3Pv5vHdfD7ic8/2eCS/ulXPO6/v9/HhE0huAKytnGuUzwG3AKyVtAt4MvLdqotGcdfIylZJMWS8FtnZDuQW8FficpH2BH9cMNlOWYpIlZ2c5cEZEPNp/MiJ2L6a5ux7G2RBJr6B8EJ1EKSUHRsQBdVMN13pWSeso2U4CnqVbNbT72RYRuyvG20PSXWNejohoZn5RpqwZqWxtclVvqLGkl1LmmZ5XN9mgZFn3AQ6JiIdqZ5mNpAMpw4tEGSLf7KqhzjpZks4DPk25cLanlFAuqm2IiI/VSzddpqwAklZQLvIC/DYi/lozzyiSNgLXziwm3WtHRcTOCrEGZMkJ5f/ViLhmxrnPR8QnamWqwWWvsiylBNJlvZJua4iI2FU7j9Uh6QDKFciV9I1kiIh1tTKNk6yYNJ9V0unA5cALI+IwSccBF0fEuytH26PvjvlQEXH/QmWZjbPOryylBNJlPRg4lOmfAffUSzRclmKSJSeApM3ApojY1B1/BVjW4oXJ+eRhnPWtpExyviBBKVlJkqwRcWHtDP+LTKUkU1ZgM2UhoW1AMxcjhukvJkCTxaQnUdYNlC+kdwNExFZJh9cMNMTMIcf9grZWD3XW+bUE+DvlffUISUe0WEo6KbJK+gJlqsEOpj4DAmguK3CmpKdnFpPKmYbJkhPK4kG3StpNWZzln4ut6IHv7Jk1QdK9DCklEXFdtVAjJMva5Mqbw0jaQvkCenffqoHbI+LouskGZckq6VcR8UZNX4nxwWhw5Vhb3EaVkgYvoGTL+hBwbES0NpdwQDda4lbKSse9YrK+bqpBGXJKWt53uB/wfcpor4sAIuKxGrlq8Z09szYsS3Q3MlPWGyR9gLKn1p4P+0bf6J+NiCc0fc/XVu9GZsm6Q9LZwFJJRwLraGxTdUlnjHs9Ir63UFlm46zzag3wqgylhFxZHwZeQJsrsAIDxeT9TBWTjZKWt/J5lSVnZwvlDq76Hld3PwG0NsJjXrnsmbUhUynJlPU/wGXAp5hahbPVN/rmi0mfLFk/QvnbP0NZPOJ24JKqiQad3j0eRJkP/ZPu+FTK77SlUuKs86f5UtInU9anKKtx3sn0z6uWph1kKSZZchIRh9XO0BIP4zRrgKTzgc9S9i3cU0oiopk3z55kWR8GXt/i6nszSXoxpZi8nfIhejtwSUQ8XTXYEJmyZiHpDuDc3nzobgGMb0TEO+omG+Sskyfpu8AqoOVSAqTLeu6w8y1OO7D5IekkBtcYuL5aoApc9swakKyUZMp6B7AmIp6qncUWjqQfMGY/xUbnFu2MiKP6jpcAO/rPtcJZJy9TKcmUNZssxSRRzhsoe8JuBZ7vTkeLFybmk4dxmrXhj5ThJhlkyvokZQjPXTR6BTpTMUmU9fLu8Qzg5cA3u+O1wN+qJJrdnZJupww3hbIARlObPvdx1gnLVJQyZJX07Yg4S9I2hrxntbhI06hiAjRVorLk7JwIvCYW+Z0t39kza4Ckm4HXAs2Wkp5kWZu/Ai3plO7p0GISERdUCTZEpqwAku6LiBNnO9eKblGRt3SH90TEzTXzjOOsk5GplCTLuiIidkk6dNjrMWRD8Nok7SRBMcmSE0DSd4B1rW8XNt9c9swakKGU9GTKmkmmYpIla/elZHVEPNwdHwZsbm0Iny1emUpJpqyjdMN41/b2iGtJlmKSJSdAN6rnOOA3TL843coolAXhYZxmDchUlDJllfQIw69AN7eYDLCvpMNnFJN9K2caJUvWC4C7u3mmAg4FPlg30nSS/s34obH7L2CcsZx18npfmGcWpV4pAZopUJmyStofOB84mLIn3I+ADwMfBR4Amit7wMuA30tqvZhkyQmwoXaAFrjsmTUgUynJlJUyXr9nGfAeYPmIf1tb88WkT4qsEXFbtzXEq7tTf2htb7CI2A9A0iXALuAGyu/0HGBFxWgDnHXyMpWSTFkpf+/HgV9S9oT7JOXvvyYittYMNsaG2gHmaEPtAHMVET+tnaEFHsZp1gBJB/Yd7iklEXFRpUgjZco6jKQtEXFC7RzDSHoRDReTflmyJlo17oGIWDXbuRY46+RIuoWpUnIaZV9AAetbKyXJsm6LiGO650sphf8Qbw+zOEj6eUScPOQOvyircTZxZ3+h+M6eWQMi4h8zTn1R0haguQKVKauk4/sOl1Du9LX8vncCU8VklaQmi0mn+azJVo17UtI5wE2UjGspq8m2yFkn5/C+UvJ12i4lmbI+23sSEc9L+kujOdMUkyw5ASLi5O5xv9pZWtDylx6zRSNTKcmUFbii7/lzwJ+As+pEGS9TMUmUNdOy22cDV3U/AfyiO9ciZ52cNKWEXFlXSfpX91zAPt2xi8n/KUtOAEnLgA8BRwAPAtdGxHN1U9XjYZxmDehWjOrplZLLI+KhOolGy5Q1k2TLWafImmnVOFucJD3P1J1GAftQ9jFtrpRkyppJlmKSJSeApG9RLk78DHgn8GhErK+bqp5Wr8abLSoRcWrtDHOVKWs3r+xMBudsXVwr0xjbKXvXZSgmWbI2v2qcpI9HxKWSrmb4wkfN7F/prJMXEUtrZ5irTFmTuY6pYvIuyj62LRaTLDmhXIzsDTm+hrL1wqLlsmfWgEylJFNW4BbgCWALfV/2G9V8MemTJeuG2gHmYGf3eF/VFHPjrGaTl6WYZMkJ04ccPyepZpbqPIzTrAGSbmOqlPTmQBERV4z8jypJlnV7RBxdO8dcSDpl2PkWl47OlNXMrGWS7o+I40cdtyJLTvCQ45lc9swakKyUZMr6NeDqiNhWO4stnDEbajf3QS/p1nGvt3S31FnNJi9LMcmS0wZ5GKdZG+6VdEySUtJ8Vknbgd2U97j3dZt/P8PUh9KxNfP1S1ZMUmTNsFpcnzcBfwZuBH5N+V22ylnNJizLXMgsOW2Q7+yZVTSjlBwJtFxKMmV9HDhu1OsR8egCxjEbqdvw+W2U/d+OBX4I3BgRO6oGG8JZzczycdkzqyhTKUmWtdm5BGajdIsfrQUuAzZGxJcrRxrJWc3McvAwTrO6HmmpJM0iU9aDJF046sWIuHIhw5iN05WR1ZRCshL4EnBzzUyjOKuZWS4ue2Z1ZSolmbIuBV6C5+lY4yRdDxwNbKbcddpeOdJIzmpmlo+HcZpVJGkX8FVGlJKI2LiwiUZLltXDOC0FSbuZWuGu/wO5qUVvwFnNzDJy2TOrKFMpSZb1dxHxuto5zMzMzGpaUjuA2SKXaZhhpqyn1Q5gZmZmVpvv7JlVJGl5RDxWO8dcZMpqZmZmZi57ZmZmZmZmeyUP4zQzMzMzM9sLueyZmZmZmZnthVz2zMzMzMzM9kIue2ZmZmZmZnuh/wLf+LJBO4y2HAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1080x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "error = labels_tip - preds\n",
    "sq_error = error * error\n",
    "avg_error = np.mean(sq_error, axis=0)\n",
    "plt.figure(figsize=(15, 10))\n",
    "plt.xticks(rotation=90)\n",
    "plt.ylabel('Prediction Error (mm)')\n",
    "bar = plt.bar(df.columns[8:][[0,1,2,12,13,14,24,25,26,36,37,38,48,49,50,60,61,62]], avg_error)\n",
    "for i in range(0,18,3):\n",
    "    bar[i].set_color('coral')\n",
    "    bar[i+1].set_color('olivedrab')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}