lstm.ipynb

{
  "cells": [
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "import numpy\nfrom keras.models import Sequential\nfrom keras.layers import Dense\nfrom keras.layers import Dropout\nfrom keras.layers import LSTM\nfrom keras.callbacks import ModelCheckpoint\nfrom keras.utils import np_utils\nfrom keras import regularizers\n# load ascii text and covert to lowercase\nfilename = \"data.txt\"\nraw_text = open(filename).read()\nraw_text = raw_text.lower()\n# create mapping of unique chars to integers\nchars = sorted(list(set(raw_text)))\nchar_to_int = dict((c, i) for i, c in enumerate(chars))\n# summarize the loaded data\nn_chars = len(raw_text)\nn_vocab = len(chars)\nprint (\"Total Characters: \", n_chars)\nprint (\"Total Vocab: \", n_vocab)\n# prepare the dataset of input to output pairs encoded as integers\nseq_length = 100\ndataX = []\ndataY = []\nfor i in range(0, n_chars - seq_length, 1):\n\tseq_in = raw_text[i:i + seq_length]\n\tseq_out = raw_text[i + seq_length]\n\tdataX.append([char_to_int[char] for char in seq_in])\n\tdataY.append(char_to_int[seq_out])\nn_patterns = len(dataX)\nprint (\"Total Patterns: \", n_patterns)\n# reshape X to be [samples, time steps, features]\nX = numpy.reshape(dataX, (n_patterns, seq_length, 1))\n# normalize\nX = X / float(n_vocab)\n# one hot encode the output variable\ny = np_utils.to_categorical(dataY)\n# define the LSTM model\nmodel = Sequential()\nmodel.add(LSTM(256, input_shape=(X.shape[1], X.shape[2])))\nmodel.add(Dropout(0.2))\nmodel.add(Dense(y.shape[1], activation='softmax'), kernel_regularizer=regularizers.l1(0.001))\nmodel.compile(loss='categorical_crossentropy', optimizer='adam')\n# define the checkpoint\nfilepath=\"weights-improvement-{epoch:02d}-{loss:.4f}.hdf5\"\ncheckpoint = ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=True, mode='min')\ncallbacks_list = [checkpoint]\n# fit the model\nmodel.fit(X, y, epochs=20, batch_size=128, callbacks=callbacks_list)",
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": "Total Characters:  833\nTotal Vocab:  27\nTotal Patterns:  733\nEpoch 1/20\n733/733 [==============================] - 9s 12ms/step - loss: 3.2525\n\nEpoch 00001: loss improved from inf to 3.25250, saving model to weights-improvement-01-3.2525.hdf5\nEpoch 2/20\n733/733 [==============================] - 6s 8ms/step - loss: 2.9577\n\nEpoch 00002: loss improved from 3.25250 to 2.95767, saving model to weights-improvement-02-2.9577.hdf5\nEpoch 3/20\n733/733 [==============================] - 6s 8ms/step - loss: 2.8726\n\nEpoch 00003: loss improved from 2.95767 to 2.87256, saving model to weights-improvement-03-2.8726.hdf5\nEpoch 4/20\n733/733 [==============================] - 6s 8ms/step - loss: 2.8657\n\nEpoch 00004: loss improved from 2.87256 to 2.86573, saving model to weights-improvement-04-2.8657.hdf5\nEpoch 5/20\n733/733 [==============================] - 6s 8ms/step - loss: 2.8447\n\nEpoch 00005: loss improved from 2.86573 to 2.84469, saving model to weights-improvement-05-2.8447.hdf5\nEpoch 6/20\n733/733 [==============================] - 6s 8ms/step - loss: 2.8624\n\nEpoch 00006: loss did not improve from 2.84469\nEpoch 7/20\n733/733 [==============================] - 6s 8ms/step - loss: 2.8420\n\nEpoch 00007: loss improved from 2.84469 to 2.84204, saving model to weights-improvement-07-2.8420.hdf5\nEpoch 8/20\n733/733 [==============================] - 7s 9ms/step - loss: 2.8349\n\nEpoch 00008: loss improved from 2.84204 to 2.83493, saving model to weights-improvement-08-2.8349.hdf5\nEpoch 9/20\n733/733 [==============================] - 10s 14ms/step - loss: 2.8360\n\nEpoch 00009: loss did not improve from 2.83493\nEpoch 10/20\n733/733 [==============================] - 7s 10ms/step - loss: 2.8411\n\nEpoch 00010: loss did not improve from 2.83493\nEpoch 11/20\n733/733 [==============================] - 7s 10ms/step - loss: 2.8435\n\nEpoch 00011: loss did not improve from 2.83493\nEpoch 12/20\n733/733 [==============================] - 7s 10ms/step - loss: 2.8392\n\nEpoch 00012: loss did not improve from 2.83493\nEpoch 13/20\n733/733 [==============================] - 7s 9ms/step - loss: 2.8358\n\nEpoch 00013: loss did not improve from 2.83493\nEpoch 14/20\n733/733 [==============================] - 6s 9ms/step - loss: 2.8382\n\nEpoch 00014: loss did not improve from 2.83493\nEpoch 15/20\n733/733 [==============================] - 7s 9ms/step - loss: 2.8318\n\nEpoch 00015: loss improved from 2.83493 to 2.83184, saving model to weights-improvement-15-2.8318.hdf5\nEpoch 16/20\n733/733 [==============================] - 6s 9ms/step - loss: 2.8328\n\nEpoch 00016: loss did not improve from 2.83184\nEpoch 17/20\n733/733 [==============================] - 7s 9ms/step - loss: 2.8302\n\nEpoch 00017: loss improved from 2.83184 to 2.83016, saving model to weights-improvement-17-2.8302.hdf5\nEpoch 18/20\n733/733 [==============================] - 6s 8ms/step - loss: 2.8258\n\nEpoch 00018: loss improved from 2.83016 to 2.82578, saving model to weights-improvement-18-2.8258.hdf5\nEpoch 19/20\n733/733 [==============================] - 6s 8ms/step - loss: 2.8370\n\nEpoch 00019: loss did not improve from 2.82578\nEpoch 20/20\n733/733 [==============================] - 6s 8ms/step - loss: 2.8314\n\nEpoch 00020: loss did not improve from 2.82578\n",
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "execution_count": 8,
          "data": {
            "text/plain": "<keras.callbacks.History at 0x7f547f453198>"
          },
          "metadata": {}
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "# pick a random seed\nimport sys\nint_to_char = dict((i, c) for i, c in enumerate(chars))\nstart = numpy.random.randint(0, len(dataX)-1)\npattern = dataX[start]\nprint (\"Seed:\")\nprint (\"\\\"\", ''.join([int_to_char[value] for value in pattern]), \"\\\"\")\n# generate characters\nfor i in range(1000):\n\tx = numpy.reshape(pattern, (1, len(pattern), 1))\n\tx = x / float(n_vocab)\n\tprediction = model.predict(x, verbose=0)\n\tindex = numpy.argmax(prediction)\n\tresult = int_to_char[index]\n\tseq_in = [int_to_char[value] for value in pattern]\n\tsys.stdout.write(result)\n\tpattern.append(index)\n\tpattern = pattern[1:len(pattern)]\nprint (\"\\nDone.\")",
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "",
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "name": "python36",
      "display_name": "Python 3.6",
      "language": "python"
    },
    "language_info": {
      "mimetype": "text/x-python",
      "nbconvert_exporter": "python",
      "name": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.6",
      "file_extension": ".py",
      "codemirror_mode": {
        "version": 3,
        "name": "ipython"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}