DSDT_Keras.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "DSDT Keras.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/nrkfeller/YCBS258/blob/master/DSDT_Keras.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "xhW0sbXbdnC0",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!pip install -q pyyaml"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "0B3_fj18q-Sz",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Install dependencies\n",
        "!apt install graphviz\n",
        "!pip install pydot pydot-ng\n",
        "!echo \"Double check with Python 3\"\n",
        "!python -c \"import pydot\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "7zwaBdZJhdPZ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras import Sequential, layers\n",
        "from tensorflow.keras.layers import Dense, Flatten\n",
        "\n",
        "print(tf.VERSION)\n",
        "print(tf.keras.__version__)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "oPCGxnvKKMgt",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Models types\n",
        "### Sequential\n",
        "Simple stack of layers\n",
        "### Functional\n",
        "Multi input, multi output, shared layers, non sequential flows\n",
        "### Model Subclassing\n",
        "Customizable, define foward pass with ```call``` method. Enables [eager execution](https://www.tensorflow.org/guide/eager#build_a_model)"
      ]
    },
    {
      "metadata": {
        "id": "T3gDzVSZM3DX",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "features = np.random.random((1000, 32))\n",
        "labels = np.random.random((1000, 10))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "W65lRhammQkO",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Sequential Model"
      ]
    },
    {
      "metadata": {
        "id": "2jJXz3MkhlsB",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# maximum simplicity\n",
        "seq_model = Sequential()\n",
        "seq_model.add(Dense(20, activation='relu', input_shape=(32,)))\n",
        "seq_model.add(Dense(20, activation='relu'))\n",
        "seq_model.add(Dense(10, activation='softmax'))\n",
        "\n",
        "seq_model.compile(optimizer=tf.train.AdamOptimizer(0.001),\n",
        "              loss='categorical_crossentropy',\n",
        "              metrics=['accuracy'])\n",
        "\n",
        "seq_model.fit(features, labels, epochs=10, batch_size=32)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "KQhkaZ28mTA9",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Functional Model\n",
        "[example](https://cdn-images-1.medium.com/max/2600/1*6hF97Upuqg_LdsqWY6n_wg.png)"
      ]
    },
    {
      "metadata": {
        "id": "VBHDpI2kKY7b",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# more full featured and flexible\n",
        "inputs = keras.Input(shape=(32,))\n",
        "x = Dense(20, activation='relu')(inputs)\n",
        "x = Dense(20, activation='relu')(x)\n",
        "outputs = Dense(10, activation='softmax')(x)\n",
        "\n",
        "func_model = keras.Model(inputs, outputs)\n",
        "func_model.compile(optimizer=tf.train.AdamOptimizer(0.001),\n",
        "              loss='categorical_crossentropy',\n",
        "              metrics=['accuracy'])\n",
        "func_model.fit(features, labels, epochs=10, batch_size=32)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "YRa8kh2-mVGf",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Model Subclassing"
      ]
    },
    {
      "metadata": {
        "id": "PA-jfeUzLdKO",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# maximally flexible and hackable\n",
        "class ScModel(keras.Model):\n",
        "  \n",
        "  def __init__(self):\n",
        "    super(ScModel, self).__init__()\n",
        "    self.dense1 = Dense(20, activation='relu')\n",
        "    self.dense2 = Dense(20, activation='relu')\n",
        "    self.dense3 = Dense(10, activation='softmax')\n",
        "    \n",
        "  def call(self, inputs):\n",
        "    x = self.dense1(inputs)\n",
        "    x = self.dense2(x)\n",
        "    return self.dense3(x)\n",
        "  \n",
        "sc_model = ScModel()\n",
        "sc_model.compile(optimizer=tf.train.AdamOptimizer(0.001),\n",
        "              loss='categorical_crossentropy',\n",
        "              metrics=['accuracy'])\n",
        "sc_model.fit(features, labels, epochs=10, batch_size=32)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "j3ItCImGoDSg",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Visualize model\n",
        "\n",
        "The summary is useful for simple models, but can be confusing for models that have multiple inputs or outputs.\n",
        "\n",
        "Keras also provides a function to create a plot of the network neural network graph that can make more complex models easier to understand.\n",
        "\n",
        "The plot_model() function in Keras will create a plot of your network. This function takes a few useful arguments:\n",
        "\n",
        "* ```model```: (required) The model that you wish to plot.\n",
        "* ```to_file```: (required) The name of the file to which to save the plot.\n",
        "* ```show_shapes```: (optional, defaults to False) Whether or not to show the output shapes of each layer.\n",
        "* ```show_layer_names```: (optional, defaults to True) Whether or not to show the name for each layer."
      ]
    },
    {
      "metadata": {
        "id": "7KmC42hroBol",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "sc_model.summary()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "q7cDLp2yp3kY",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "[fully connected NN](https://www.oreilly.com/library/view/tensorflow-for-deep/9781491980446/assets/tfdl_0402.png)"
      ]
    },
    {
      "metadata": {
        "id": "Wrzmlp4npIXK",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "mod = func_model\n",
        "\n",
        "tf.keras.utils.plot_model(\n",
        "    mod, \n",
        "    to_file='{}.png'.format(mod), \n",
        "    show_shapes=True, \n",
        "    show_layer_names=True\n",
        ")\n",
        "\n",
        "from IPython.display import Image\n",
        "Image(retina=True, filename='{}.png'.format(mod))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "90NHWXygtgu0",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Evaluate and predict\n",
        "\n",
        "The `tf.keras.Model.evaluate` and `tf.keras.Model.predict` methods can use NumPy\n",
        "data and a `tf.data.Dataset`.\n",
        "\n",
        "To *evaluate* the inference-mode loss and metrics for the data provided:"
      ]
    },
    {
      "metadata": {
        "id": "fW5GZmB_jI_w",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Unseen data\n",
        "new_data  = np.random.random((1000, 32))\n",
        "new_labels = np.random.random((1000, 10))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "TnbrMxQMjjN2",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "results = sc_model.predict(new_data)\n",
        "results[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "H6tOtNLwjpje",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "sc_model.evaluate(new_data, new_labels)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "P3RAov0piqQr",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Input tf.data datasets\n",
        "\n",
        "Use the Dataset API to scale to large datasets\n",
        "or multi-device training. Pass a `tf.data.Dataset` instance to the `fit`\n",
        "method:\n",
        "\n",
        "Here, the `fit` method uses the `steps_per_epoch` argument—this is the number of\n",
        "training steps the model runs before it moves to the next epoch. Since the\n",
        "`Dataset` yields batches of data, this snippet does not require a `batch_size`."
      ]
    },
    {
      "metadata": {
        "id": "j_f1u0KyL383",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "dataset = tf.data.Dataset.from_tensor_slices((features, labels))\n",
        "dataset = dataset.batch(32)\n",
        "dataset = dataset.repeat()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "K14bKV7SiXzo",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "sc_model.fit(dataset, epochs=10, steps_per_epoch=30)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "jJSxjNQqk6B8",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Custom layers\n",
        "\n",
        "[Arguments](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Dense)\n",
        "-\n",
        "[Implementation](https://github.com/keras-team/keras/blob/master/keras/layers/core.py)\n",
        "\n",
        "Create a custom layer by subclassing `tf.keras.layers.Layer` and implementing\n",
        "the following methods:\n",
        "\n",
        "* `build`: Create the weights of the layer. Add weights with the `add_weight`\n",
        "  method.\n",
        "* `call`: Define the forward pass.\n",
        "* `compute_output_shape`: Specify how to compute the output shape of the layer\n",
        "  given the input shape.\n",
        "* Optionally, a layer can be serialized by implementing the `get_config` method\n",
        "  and the `from_config` class method.\n",
        "\n",
        "Here's an example of a custom layer that implements a `matmul` of an input with\n",
        "a kernel matrix:"
      ]
    },
    {
      "metadata": {
        "id": "vd_n_oj8ktYW",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "layers."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "RpoNJ20KihgZ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "class MyLayer(layers.Layer):\n",
        "\n",
        "  def __init__(self, output_dim, **kwargs):\n",
        "    self.output_dim = output_dim\n",
        "    super(MyLayer, self).__init__(**kwargs)\n",
        "\n",
        "  def build(self, input_shape):\n",
        "    shape = tf.TensorShape((input_shape[1], self.output_dim))\n",
        "    # Create a trainable weight variable for this layer.\n",
        "    self.kernel = self.add_weight(name='kernel',\n",
        "                                  shape=shape,\n",
        "                                  initializer='uniform',\n",
        "                                  trainable=True)\n",
        "    # Make sure to call the `build` method at the end\n",
        "    super(MyLayer, self).build(input_shape)\n",
        "\n",
        "  def call(self, inputs):\n",
        "    return tf.matmul(inputs, self.kernel)\n",
        "\n",
        "  def compute_output_shape(self, input_shape):\n",
        "    shape = tf.TensorShape(input_shape).as_list()\n",
        "    shape[-1] = self.output_dim\n",
        "    return tf.TensorShape(shape)\n",
        "\n",
        "  def get_config(self):\n",
        "    base_config = super(MyLayer, self).get_config()\n",
        "    base_config['output_dim'] = self.output_dim\n",
        "    return base_config\n",
        "\n",
        "  @classmethod\n",
        "  def from_config(cls, config):\n",
        "    return cls(**config)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "xemR242ak-yV",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "model = Sequential()\n",
        "\n",
        "model.add(MyLayer(10))\n",
        "model.add(layers.Activation('softmax'))\n",
        "\n",
        "model.compile(optimizer=tf.train.RMSPropOptimizer(0.001),\n",
        "              loss='categorical_crossentropy',\n",
        "              metrics=['accuracy'])\n",
        "\n",
        "model.fit(features, labels, batch_size=32, epochs=5)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "IwzvqjrLlJrZ",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Callbacks\n",
        "\n",
        "A callback is an object passed to a model to customize and extend its behavior\n",
        "during training. You can write your own custom callback, or use the built-in\n",
        "`tf.keras.callbacks` that include:\n",
        "\n",
        "* `tf.keras.callbacks.ModelCheckpoint`: Save checkpoints of your model at\n",
        "  regular intervals.\n",
        "* `tf.keras.callbacks.LearningRateScheduler`: Dynamically change the learning\n",
        "  rate.\n",
        "* `tf.keras.callbacks.EarlyStopping`: Interrupt training when validation\n",
        "  performance has stopped improving.\n",
        "* `tf.keras.callbacks.TensorBoard`: Monitor the model's behavior using\n",
        "  [TensorBoard](./summaries_and_tensorboard.md).\n",
        "\n",
        "To use a `tf.keras.callbacks.Callback`, pass it to the model's `fit` method:"
      ]
    },
    {
      "metadata": {
        "id": "SkXcbKEflI7W",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "callbacks = [\n",
        "  tf.keras.callbacks.EarlyStopping(patience=2, monitor='val_loss'),\n",
        "  tf.keras.callbacks.TensorBoard(log_dir='./logs')\n",
        "]\n",
        "model.fit(features, labels, batch_size=32, epochs=5, callbacks=callbacks,\n",
        "          validation_data=(new_data, new_labels))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "31yq6mUxlZxN",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Model Saving and Restoring\n",
        "### Weights only\n",
        "\n",
        "Save and load the weights of a model using `tf.keras.Model.save_weights`:\n",
        "\n",
        "### Configuration only\n",
        "\n",
        "A model's configuration can be saved—this serializes the model architecture\n",
        "without any weights. A saved configuration can recreate and initialize the same\n",
        "model, even without the code that defined the original model. Keras supports\n",
        "JSON and YAML serialization formats:\n",
        "\n",
        "### Entire model\n",
        "\n",
        "The entire model can be saved to a file that contains the weight values, the\n",
        "model's configuration, and even the optimizer's configuration. This allows you\n",
        "to checkpoint a model and resume training later—from the exact same\n",
        "state—without access to the original code."
      ]
    },
    {
      "metadata": {
        "id": "af1xTnLMlUwS",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "model = tf.keras.Sequential([\n",
        "  layers.Dense(10, activation='softmax', input_shape=(32,)),\n",
        "  layers.Dense(10, activation='softmax')\n",
        "])\n",
        "\n",
        "model.compile(optimizer='rmsprop',\n",
        "              loss='categorical_crossentropy',\n",
        "              metrics=['accuracy'])\n",
        "model.fit(features, labels, batch_size=32, epochs=5)\n",
        "\n",
        "model.save('my_model.h5')\n",
        "\n",
        "model = tf.keras.models.load_model('my_model.h5')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "GrpOFkPjhXYD",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Pretrained Models\n",
        "\n",
        "### Use pretrained model\n",
        "### Get embeddings from pretrained models\n",
        "### Finetune pretrained model"
      ]
    },
    {
      "metadata": {
        "id": "FjEmUrRGlE3U",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from tensorflow.keras.applications.resnet50 import ResNet50\n",
        "from tensorflow.keras.preprocessing import image\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.layers import Dropout, Flatten, Dense\n",
        "from tensorflow.keras.applications.resnet50 import preprocess_input, decode_predictions\n",
        "\n",
        "import tensorflow.keras.backend as K\n",
        "import numpy as np"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "jtAYAEHXKBGL",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "model = ResNet50(weights='imagenet')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "aAoBgbX-Hoz0",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "[pretrained keras models](https://keras.io/applications/)\n",
        "\n",
        "[imagenet classes](https://gist.github.com/yrevar/942d3a0ac09ec9e5eb3a)"
      ]
    },
    {
      "metadata": {
        "id": "LaJ_AvtzjzyX",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#!wget -O test.png https://www.freepngimg.com/thumb/corn/23-corn-png-image-thumb.png\n",
        "!wget -O test.png https://vignette.wikia.nocookie.net/dino/images/f/f6/JW_triceratops.png/revision/latest?cb=20150407211112"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "tvGO9WyqjmWG",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "img_path = 'test.png'\n",
        "img = image.load_img(img_path, target_size=(224, 224))\n",
        "x = image.img_to_array(img)\n",
        "x = np.expand_dims(x, axis=0)\n",
        "x = preprocess_input(x)\n",
        "\n",
        "preds = model.predict(x)\n",
        "\n",
        "print('Predicted:', decode_predictions(preds, top=3)[0])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "QF5Gpp-rTzcx",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "31Vs2mDAJAEP",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Get the embedding form the corn picture\n",
        "get_last_layer_output = K.function(\n",
        "    [model.layers[0].input],\n",
        "    [model.layers[-1].output]\n",
        ")\n",
        "\n",
        "get_last_layer_output([x])[0].shape"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "y1VnAZHlUeDE",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Other way to do the same thing, just create another model\n",
        "emb_model = Model(inputs=model.inputs, outputs=model.get_layer('avg_pool').output)\n",
        "\n",
        "emb_model.predict(x).shape"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "OZmDklyGNuSq",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Fine-tuning pre-trained model\n",
        "Provided that our dataset is not drastically different in context to the original dataset, we can use pretrained models like the one above to fine tune to a different task"
      ]
    },
    {
      "metadata": {
        "id": "cI6x46u9NtrV",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "[print(l) for l in model.layers[-10:]]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ERzr3uRMQRzb",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# remove the softmax layer\n",
        "model = Model(inputs=model.inputs, outputs=model.get_layer('avg_pool').output)\n",
        "[print(l) for l in model.layers[-10:]]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "qvcxlZokKWMs",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Freeze the previous layers\n",
        "for layer in model.layers[:-1]:\n",
        "    layer.trainable = False\n",
        "\n",
        "# add a couple layers\n",
        "x = Flatten()(model.layers[-1].output)\n",
        "x = Dense(1000, activation='relu')(x)\n",
        "x = Dropout(0.5)(x)\n",
        "x = Dense(500, activation='relu')(x)\n",
        "x = Dropout(0.5)(x)\n",
        "out = Dense(100, activation='softmax')(x)\n",
        "\n",
        "model = Model(inputs=model.input, outputs=out)\n",
        "\n",
        "[print(l) for l in model.layers[-10:]]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "fJIqzEuHUol1",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "wCGw3jUNSlFo",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "model.compile(optimizer=Adam(lr=0.0001, momentum=0.9), loss='caregorical_crossentropy')\n",
        "model.fit_generator(...,...)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "IsDT1y9mQ936",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Toy video-QA problem\n",
        "![sequence](https://storage.googleapis.com/nicksdemobucket/Screen%20Shot%202019-02-20%20at%202.17.39%20PM.png)\n",
        "![architecture](https://storage.googleapis.com/nicksdemobucket/Screen%20Shot%202019-02-20%20at%202.17.52%20PM.png)"
      ]
    },
    {
      "metadata": {
        "id": "SNDEpxR8Q-Oq",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from tensorflow import keras\n",
        "from tensorflow.keras import layers\n",
        "from tensorflow.keras.applications import InceptionV3"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "P9VtSx3-XutP",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "input_voc_size = # Size of vocabulary of possible answers\n",
        "data_generator = # input as question and "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "JEzLGoJZXiRd",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "video = keras.Input(shape=(None, 150, 150, 3), name='video')\n",
        "cnn = InceptionV3(weights='imagenet',\n",
        "include_top=False,\n",
        "pooling='avg')\n",
        "cnn.trainable = False\n",
        "frame_features = layers.TimeDistributed(cnn)(video)\n",
        "video_vector = layers.LSTM(256)(frame_features)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "FmqQ20GVRFaU",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "question = keras.Input(shape=(None,), dtype='int32', name='question')\n",
        "embedded_words = layers.Embedding(input_voc_size, 256)(question)\n",
        "question_vector = layers.LSTM(128)(embedded_words)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Vw4fNeaWRNDL",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "x = layers.concatenate([video_vector, question_vector])\n",
        "x = layers.Dense(128, activation=tf.nn.relu)(x)\n",
        "predictions = layers.Dense(output_voc_size,\n",
        " activation='softmax',\n",
        " name='predictions')(x)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "4pStrrubRRCi",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "model = keras.models.Model([video, question], predictions)\n",
        "model.compile(optimizer=tf.AdamOptimizer(),loss=keras.losses.categorical_crossentropy)\n",
        "model.fit_generator(data_generator, steps_per_epoch=1000, epochs=100)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "dUrsPbBabv1x",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Futher Reading\n",
        "* https://github.com/Dataweekends/zero_to_deep_learning_video \n",
        "* https://github.com/keras-team/keras/tree/master/examples \n",
        "* https://github.com/PacktPublishing/Advanced-Deep-Learning-with-Keras \n",
        "* https://medium.com/tensorflow/tagged/keras\n",
        "* https://github.com/keras-team/keras"
      ]
    }
  ]
}