pytorch-mnist-VAE.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prerequisites\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "from torch.autograd import Variable\n",
    "from torchvision.utils import save_image\n",
    "\n",
    "bs = 100\n",
    "# MNIST Dataset\n",
    "train_dataset = datasets.MNIST(root='./mnist_data/', train=True, transform=transforms.ToTensor(), download=True)\n",
    "test_dataset = datasets.MNIST(root='./mnist_data/', train=False, transform=transforms.ToTensor(), download=False)\n",
    "\n",
    "# Data Loader (Input Pipeline)\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=bs, shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=bs, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VAE(nn.Module):\n",
    "    def __init__(self, x_dim, h_dim1, h_dim2, z_dim):\n",
    "        super(VAE, self).__init__()\n",
    "        \n",
    "        # encoder part\n",
    "        self.fc1 = nn.Linear(x_dim, h_dim1)\n",
    "        self.fc2 = nn.Linear(h_dim1, h_dim2)\n",
    "        self.fc31 = nn.Linear(h_dim2, z_dim)\n",
    "        self.fc32 = nn.Linear(h_dim2, z_dim)\n",
    "        # decoder part\n",
    "        self.fc4 = nn.Linear(z_dim, h_dim2)\n",
    "        self.fc5 = nn.Linear(h_dim2, h_dim1)\n",
    "        self.fc6 = nn.Linear(h_dim1, x_dim)\n",
    "        \n",
    "    def encoder(self, x):\n",
    "        h = F.relu(self.fc1(x))\n",
    "        h = F.relu(self.fc2(h))\n",
    "        return self.fc31(h), self.fc32(h) # mu, log_var\n",
    "    \n",
    "    def sampling(self, mu, log_var):\n",
    "        std = torch.exp(0.5*log_var)\n",
    "        eps = torch.randn_like(std)\n",
    "        return eps.mul(std).add_(mu) # return z sample\n",
    "        \n",
    "    def decoder(self, z):\n",
    "        h = F.relu(self.fc4(z))\n",
    "        h = F.relu(self.fc5(h))\n",
    "        return F.sigmoid(self.fc6(h)) \n",
    "    \n",
    "    def forward(self, x):\n",
    "        mu, log_var = self.encoder(x.view(-1, 784))\n",
    "        z = self.sampling(mu, log_var)\n",
    "        return self.decoder(z), mu, log_var\n",
    "\n",
    "# build model\n",
    "vae = VAE(x_dim=784, h_dim1= 512, h_dim2=256, z_dim=2)\n",
    "if torch.cuda.is_available():\n",
    "    vae.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VAE(\n",
       "  (fc1): Linear(in_features=784, out_features=512, bias=True)\n",
       "  (fc2): Linear(in_features=512, out_features=256, bias=True)\n",
       "  (fc31): Linear(in_features=256, out_features=2, bias=True)\n",
       "  (fc32): Linear(in_features=256, out_features=2, bias=True)\n",
       "  (fc4): Linear(in_features=2, out_features=256, bias=True)\n",
       "  (fc5): Linear(in_features=256, out_features=512, bias=True)\n",
       "  (fc6): Linear(in_features=512, out_features=784, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vae"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.Adam(vae.parameters())\n",
    "# return reconstruction error + KL divergence losses\n",
    "def loss_function(recon_x, x, mu, log_var):\n",
    "    BCE = F.binary_cross_entropy(recon_x, x.view(-1, 784), reduction='sum')\n",
    "    KLD = -0.5 * torch.sum(1 + log_var - mu.pow(2) - log_var.exp())\n",
    "    return BCE + KLD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(epoch):\n",
    "    vae.train()\n",
    "    train_loss = 0\n",
    "    for batch_idx, (data, _) in enumerate(train_loader):\n",
    "        data = data.cuda()\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        recon_batch, mu, log_var = vae(data)\n",
    "        loss = loss_function(recon_batch, data, mu, log_var)\n",
    "        \n",
    "        loss.backward()\n",
    "        train_loss += loss.item()\n",
    "        optimizer.step()\n",
    "        \n",
    "        if batch_idx % 100 == 0:\n",
    "            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n",
    "                epoch, batch_idx * len(data), len(train_loader.dataset),\n",
    "                100. * batch_idx / len(train_loader), loss.item() / len(data)))\n",
    "    print('====> Epoch: {} Average loss: {:.4f}'.format(epoch, train_loss / len(train_loader.dataset)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test():\n",
    "    vae.eval()\n",
    "    test_loss= 0\n",
    "    with torch.no_grad():\n",
    "        for data, _ in test_loader:\n",
    "            data = data.cuda()\n",
    "            recon, mu, log_var = vae(data)\n",
    "            \n",
    "            # sum up batch loss\n",
    "            test_loss += loss_function(recon, data, mu, log_var).item()\n",
    "        \n",
    "    test_loss /= len(test_loader.dataset)\n",
    "    print('====> Test set loss: {:.4f}'.format(test_loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lyeoni\\AppData\\Local\\Continuum\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\torch\\nn\\functional.py:1006: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\n",
      "  warnings.warn(\"nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 1 [0/60000 (0%)]\tLoss: 544.540078\n",
      "Train Epoch: 1 [10000/60000 (17%)]\tLoss: 184.232109\n",
      "Train Epoch: 1 [20000/60000 (33%)]\tLoss: 162.313955\n",
      "Train Epoch: 1 [30000/60000 (50%)]\tLoss: 165.958750\n",
      "Train Epoch: 1 [40000/60000 (67%)]\tLoss: 159.636836\n",
      "Train Epoch: 1 [50000/60000 (83%)]\tLoss: 157.480146\n",
      "====> Epoch: 1 Average loss: 178.0764\n",
      "====> Test set loss: 162.4080\n",
      "Train Epoch: 2 [0/60000 (0%)]\tLoss: 165.543457\n",
      "Train Epoch: 2 [10000/60000 (17%)]\tLoss: 169.160371\n",
      "Train Epoch: 2 [20000/60000 (33%)]\tLoss: 156.650479\n",
      "Train Epoch: 2 [30000/60000 (50%)]\tLoss: 157.932402\n",
      "Train Epoch: 2 [40000/60000 (67%)]\tLoss: 149.316348\n",
      "Train Epoch: 2 [50000/60000 (83%)]\tLoss: 152.364521\n",
      "====> Epoch: 2 Average loss: 157.7688\n",
      "====> Test set loss: 154.8181\n",
      "Train Epoch: 3 [0/60000 (0%)]\tLoss: 156.378076\n",
      "Train Epoch: 3 [10000/60000 (17%)]\tLoss: 152.164893\n",
      "Train Epoch: 3 [20000/60000 (33%)]\tLoss: 149.615713\n",
      "Train Epoch: 3 [30000/60000 (50%)]\tLoss: 156.538242\n",
      "Train Epoch: 3 [40000/60000 (67%)]\tLoss: 159.814023\n",
      "Train Epoch: 3 [50000/60000 (83%)]\tLoss: 152.775078\n",
      "====> Epoch: 3 Average loss: 152.4924\n",
      "====> Test set loss: 150.5631\n",
      "Train Epoch: 4 [0/60000 (0%)]\tLoss: 147.074658\n",
      "Train Epoch: 4 [10000/60000 (17%)]\tLoss: 149.979785\n",
      "Train Epoch: 4 [20000/60000 (33%)]\tLoss: 148.332109\n",
      "Train Epoch: 4 [30000/60000 (50%)]\tLoss: 151.918740\n",
      "Train Epoch: 4 [40000/60000 (67%)]\tLoss: 150.095313\n",
      "Train Epoch: 4 [50000/60000 (83%)]\tLoss: 148.829180\n",
      "====> Epoch: 4 Average loss: 149.4004\n",
      "====> Test set loss: 148.6881\n",
      "Train Epoch: 5 [0/60000 (0%)]\tLoss: 156.652500\n",
      "Train Epoch: 5 [10000/60000 (17%)]\tLoss: 160.000967\n",
      "Train Epoch: 5 [20000/60000 (33%)]\tLoss: 144.564707\n",
      "Train Epoch: 5 [30000/60000 (50%)]\tLoss: 141.268037\n",
      "Train Epoch: 5 [40000/60000 (67%)]\tLoss: 150.177188\n",
      "Train Epoch: 5 [50000/60000 (83%)]\tLoss: 146.934717\n",
      "====> Epoch: 5 Average loss: 147.2922\n",
      "====> Test set loss: 147.3806\n",
      "Train Epoch: 6 [0/60000 (0%)]\tLoss: 137.584609\n",
      "Train Epoch: 6 [10000/60000 (17%)]\tLoss: 143.717187\n",
      "Train Epoch: 6 [20000/60000 (33%)]\tLoss: 151.873164\n",
      "Train Epoch: 6 [30000/60000 (50%)]\tLoss: 149.592031\n",
      "Train Epoch: 6 [40000/60000 (67%)]\tLoss: 152.565313\n",
      "Train Epoch: 6 [50000/60000 (83%)]\tLoss: 143.237783\n",
      "====> Epoch: 6 Average loss: 145.8375\n",
      "====> Test set loss: 145.7968\n",
      "Train Epoch: 7 [0/60000 (0%)]\tLoss: 154.753066\n",
      "Train Epoch: 7 [10000/60000 (17%)]\tLoss: 140.587461\n",
      "Train Epoch: 7 [20000/60000 (33%)]\tLoss: 139.729072\n",
      "Train Epoch: 7 [30000/60000 (50%)]\tLoss: 136.815674\n",
      "Train Epoch: 7 [40000/60000 (67%)]\tLoss: 144.345664\n",
      "Train Epoch: 7 [50000/60000 (83%)]\tLoss: 133.049941\n",
      "====> Epoch: 7 Average loss: 144.7191\n",
      "====> Test set loss: 144.7703\n",
      "Train Epoch: 8 [0/60000 (0%)]\tLoss: 135.705625\n",
      "Train Epoch: 8 [10000/60000 (17%)]\tLoss: 140.541279\n",
      "Train Epoch: 8 [20000/60000 (33%)]\tLoss: 149.972246\n",
      "Train Epoch: 8 [30000/60000 (50%)]\tLoss: 140.558301\n",
      "Train Epoch: 8 [40000/60000 (67%)]\tLoss: 144.370586\n",
      "Train Epoch: 8 [50000/60000 (83%)]\tLoss: 140.096621\n",
      "====> Epoch: 8 Average loss: 143.7319\n",
      "====> Test set loss: 144.1646\n",
      "Train Epoch: 9 [0/60000 (0%)]\tLoss: 144.310107\n",
      "Train Epoch: 9 [10000/60000 (17%)]\tLoss: 143.483672\n",
      "Train Epoch: 9 [20000/60000 (33%)]\tLoss: 139.612285\n",
      "Train Epoch: 9 [30000/60000 (50%)]\tLoss: 142.273838\n",
      "Train Epoch: 9 [40000/60000 (67%)]\tLoss: 141.792080\n",
      "Train Epoch: 9 [50000/60000 (83%)]\tLoss: 146.816553\n",
      "====> Epoch: 9 Average loss: 143.0411\n",
      "====> Test set loss: 143.7913\n",
      "Train Epoch: 10 [0/60000 (0%)]\tLoss: 139.106621\n",
      "Train Epoch: 10 [10000/60000 (17%)]\tLoss: 146.593350\n",
      "Train Epoch: 10 [20000/60000 (33%)]\tLoss: 144.646611\n",
      "Train Epoch: 10 [30000/60000 (50%)]\tLoss: 146.716641\n",
      "Train Epoch: 10 [40000/60000 (67%)]\tLoss: 142.536992\n",
      "Train Epoch: 10 [50000/60000 (83%)]\tLoss: 143.560186\n",
      "====> Epoch: 10 Average loss: 142.4858\n",
      "====> Test set loss: 143.7057\n",
      "Train Epoch: 11 [0/60000 (0%)]\tLoss: 146.276572\n",
      "Train Epoch: 11 [10000/60000 (17%)]\tLoss: 132.048018\n",
      "Train Epoch: 11 [20000/60000 (33%)]\tLoss: 144.678604\n",
      "Train Epoch: 11 [30000/60000 (50%)]\tLoss: 135.977881\n",
      "Train Epoch: 11 [40000/60000 (67%)]\tLoss: 147.658887\n",
      "Train Epoch: 11 [50000/60000 (83%)]\tLoss: 140.258711\n",
      "====> Epoch: 11 Average loss: 141.9173\n",
      "====> Test set loss: 143.2985\n",
      "Train Epoch: 12 [0/60000 (0%)]\tLoss: 148.664502\n",
      "Train Epoch: 12 [10000/60000 (17%)]\tLoss: 148.922070\n",
      "Train Epoch: 12 [20000/60000 (33%)]\tLoss: 143.017979\n",
      "Train Epoch: 12 [30000/60000 (50%)]\tLoss: 140.038389\n",
      "Train Epoch: 12 [40000/60000 (67%)]\tLoss: 144.384346\n",
      "Train Epoch: 12 [50000/60000 (83%)]\tLoss: 148.121699\n",
      "====> Epoch: 12 Average loss: 141.2700\n",
      "====> Test set loss: 142.4564\n",
      "Train Epoch: 13 [0/60000 (0%)]\tLoss: 138.919609\n",
      "Train Epoch: 13 [10000/60000 (17%)]\tLoss: 146.280313\n",
      "Train Epoch: 13 [20000/60000 (33%)]\tLoss: 133.725488\n",
      "Train Epoch: 13 [30000/60000 (50%)]\tLoss: 145.125605\n",
      "Train Epoch: 13 [40000/60000 (67%)]\tLoss: 141.544844\n",
      "Train Epoch: 13 [50000/60000 (83%)]\tLoss: 143.635371\n",
      "====> Epoch: 13 Average loss: 140.9097\n",
      "====> Test set loss: 142.0727\n",
      "Train Epoch: 14 [0/60000 (0%)]\tLoss: 143.550967\n",
      "Train Epoch: 14 [10000/60000 (17%)]\tLoss: 139.760225\n",
      "Train Epoch: 14 [20000/60000 (33%)]\tLoss: 148.380859\n",
      "Train Epoch: 14 [30000/60000 (50%)]\tLoss: 150.115967\n",
      "Train Epoch: 14 [40000/60000 (67%)]\tLoss: 137.295576\n",
      "Train Epoch: 14 [50000/60000 (83%)]\tLoss: 142.828633\n",
      "====> Epoch: 14 Average loss: 140.4962\n",
      "====> Test set loss: 141.6135\n",
      "Train Epoch: 15 [0/60000 (0%)]\tLoss: 139.083018\n",
      "Train Epoch: 15 [10000/60000 (17%)]\tLoss: 139.302393\n",
      "Train Epoch: 15 [20000/60000 (33%)]\tLoss: 137.078604\n",
      "Train Epoch: 15 [30000/60000 (50%)]\tLoss: 141.197686\n",
      "Train Epoch: 15 [40000/60000 (67%)]\tLoss: 133.740586\n",
      "Train Epoch: 15 [50000/60000 (83%)]\tLoss: 139.992148\n",
      "====> Epoch: 15 Average loss: 140.1263\n",
      "====> Test set loss: 141.6311\n",
      "Train Epoch: 16 [0/60000 (0%)]\tLoss: 133.581699\n",
      "Train Epoch: 16 [10000/60000 (17%)]\tLoss: 138.739521\n",
      "Train Epoch: 16 [20000/60000 (33%)]\tLoss: 136.517959\n",
      "Train Epoch: 16 [30000/60000 (50%)]\tLoss: 145.232949\n",
      "Train Epoch: 16 [40000/60000 (67%)]\tLoss: 134.824375\n",
      "Train Epoch: 16 [50000/60000 (83%)]\tLoss: 145.148975\n",
      "====> Epoch: 16 Average loss: 139.9088\n",
      "====> Test set loss: 141.8304\n",
      "Train Epoch: 17 [0/60000 (0%)]\tLoss: 137.299766\n",
      "Train Epoch: 17 [10000/60000 (17%)]\tLoss: 147.352148\n",
      "Train Epoch: 17 [20000/60000 (33%)]\tLoss: 140.070234\n",
      "Train Epoch: 17 [30000/60000 (50%)]\tLoss: 133.969385\n",
      "Train Epoch: 17 [40000/60000 (67%)]\tLoss: 132.779844\n",
      "Train Epoch: 17 [50000/60000 (83%)]\tLoss: 139.806924\n",
      "====> Epoch: 17 Average loss: 139.7195\n",
      "====> Test set loss: 141.5800\n",
      "Train Epoch: 18 [0/60000 (0%)]\tLoss: 147.979365\n",
      "Train Epoch: 18 [10000/60000 (17%)]\tLoss: 141.225674\n",
      "Train Epoch: 18 [20000/60000 (33%)]\tLoss: 143.228125\n",
      "Train Epoch: 18 [30000/60000 (50%)]\tLoss: 144.623408\n",
      "Train Epoch: 18 [40000/60000 (67%)]\tLoss: 130.182588\n",
      "Train Epoch: 18 [50000/60000 (83%)]\tLoss: 134.544248\n",
      "====> Epoch: 18 Average loss: 139.4170\n",
      "====> Test set loss: 140.8149\n",
      "Train Epoch: 19 [0/60000 (0%)]\tLoss: 141.852832\n",
      "Train Epoch: 19 [10000/60000 (17%)]\tLoss: 131.704590\n",
      "Train Epoch: 19 [20000/60000 (33%)]\tLoss: 136.412021\n",
      "Train Epoch: 19 [30000/60000 (50%)]\tLoss: 140.386006\n",
      "Train Epoch: 19 [40000/60000 (67%)]\tLoss: 133.427041\n",
      "Train Epoch: 19 [50000/60000 (83%)]\tLoss: 136.500303\n",
      "====> Epoch: 19 Average loss: 139.0172\n",
      "====> Test set loss: 140.5691\n",
      "Train Epoch: 20 [0/60000 (0%)]\tLoss: 136.212500\n",
      "Train Epoch: 20 [10000/60000 (17%)]\tLoss: 137.772354\n",
      "Train Epoch: 20 [20000/60000 (33%)]\tLoss: 143.806846\n",
      "Train Epoch: 20 [30000/60000 (50%)]\tLoss: 133.786084\n",
      "Train Epoch: 20 [40000/60000 (67%)]\tLoss: 134.133262\n",
      "Train Epoch: 20 [50000/60000 (83%)]\tLoss: 133.471270\n",
      "====> Epoch: 20 Average loss: 138.6413\n",
      "====> Test set loss: 140.8972\n",
      "Train Epoch: 21 [0/60000 (0%)]\tLoss: 136.250059\n",
      "Train Epoch: 21 [10000/60000 (17%)]\tLoss: 144.601562\n",
      "Train Epoch: 21 [20000/60000 (33%)]\tLoss: 134.401846\n",
      "Train Epoch: 21 [30000/60000 (50%)]\tLoss: 136.697490\n",
      "Train Epoch: 21 [40000/60000 (67%)]\tLoss: 135.355898\n",
      "Train Epoch: 21 [50000/60000 (83%)]\tLoss: 141.310078\n",
      "====> Epoch: 21 Average loss: 138.6225\n",
      "====> Test set loss: 140.6093\n",
      "Train Epoch: 22 [0/60000 (0%)]\tLoss: 135.936953\n",
      "Train Epoch: 22 [10000/60000 (17%)]\tLoss: 136.139209\n",
      "Train Epoch: 22 [20000/60000 (33%)]\tLoss: 138.357227\n",
      "Train Epoch: 22 [30000/60000 (50%)]\tLoss: 137.317656\n",
      "Train Epoch: 22 [40000/60000 (67%)]\tLoss: 139.188086\n",
      "Train Epoch: 22 [50000/60000 (83%)]\tLoss: 135.284033\n",
      "====> Epoch: 22 Average loss: 138.5469\n",
      "====> Test set loss: 140.5935\n",
      "Train Epoch: 23 [0/60000 (0%)]\tLoss: 139.954746\n",
      "Train Epoch: 23 [10000/60000 (17%)]\tLoss: 140.082803\n",
      "Train Epoch: 23 [20000/60000 (33%)]\tLoss: 137.566084\n",
      "Train Epoch: 23 [30000/60000 (50%)]\tLoss: 134.860957\n",
      "Train Epoch: 23 [40000/60000 (67%)]\tLoss: 136.043418\n",
      "Train Epoch: 23 [50000/60000 (83%)]\tLoss: 130.924062\n",
      "====> Epoch: 23 Average loss: 138.1881\n",
      "====> Test set loss: 140.4619\n",
      "Train Epoch: 24 [0/60000 (0%)]\tLoss: 134.665166\n",
      "Train Epoch: 24 [10000/60000 (17%)]\tLoss: 132.645508\n",
      "Train Epoch: 24 [20000/60000 (33%)]\tLoss: 136.634199\n",
      "Train Epoch: 24 [30000/60000 (50%)]\tLoss: 143.385332\n",
      "Train Epoch: 24 [40000/60000 (67%)]\tLoss: 135.292627\n",
      "Train Epoch: 24 [50000/60000 (83%)]\tLoss: 141.106641\n",
      "====> Epoch: 24 Average loss: 138.1694\n",
      "====> Test set loss: 139.9362\n",
      "Train Epoch: 25 [0/60000 (0%)]\tLoss: 127.883643\n",
      "Train Epoch: 25 [10000/60000 (17%)]\tLoss: 141.906982\n",
      "Train Epoch: 25 [20000/60000 (33%)]\tLoss: 137.590957\n",
      "Train Epoch: 25 [30000/60000 (50%)]\tLoss: 131.470303\n",
      "Train Epoch: 25 [40000/60000 (67%)]\tLoss: 139.002139\n",
      "Train Epoch: 25 [50000/60000 (83%)]\tLoss: 141.467363\n",
      "====> Epoch: 25 Average loss: 137.7709\n",
      "====> Test set loss: 140.1430\n",
      "Train Epoch: 26 [0/60000 (0%)]\tLoss: 137.514336\n",
      "Train Epoch: 26 [10000/60000 (17%)]\tLoss: 136.857090\n",
      "Train Epoch: 26 [20000/60000 (33%)]\tLoss: 141.599072\n",
      "Train Epoch: 26 [30000/60000 (50%)]\tLoss: 136.554941\n",
      "Train Epoch: 26 [40000/60000 (67%)]\tLoss: 136.909824\n",
      "Train Epoch: 26 [50000/60000 (83%)]\tLoss: 133.868838\n",
      "====> Epoch: 26 Average loss: 137.4966\n",
      "====> Test set loss: 140.0211\n",
      "Train Epoch: 27 [0/60000 (0%)]\tLoss: 138.974111\n",
      "Train Epoch: 27 [10000/60000 (17%)]\tLoss: 137.810566\n",
      "Train Epoch: 27 [20000/60000 (33%)]\tLoss: 140.517969\n",
      "Train Epoch: 27 [30000/60000 (50%)]\tLoss: 135.388936\n",
      "Train Epoch: 27 [40000/60000 (67%)]\tLoss: 132.792246\n",
      "Train Epoch: 27 [50000/60000 (83%)]\tLoss: 135.815537\n",
      "====> Epoch: 27 Average loss: 137.1959\n",
      "====> Test set loss: 139.5045\n",
      "Train Epoch: 28 [0/60000 (0%)]\tLoss: 139.144697\n",
      "Train Epoch: 28 [10000/60000 (17%)]\tLoss: 139.859043\n",
      "Train Epoch: 28 [20000/60000 (33%)]\tLoss: 137.748906\n",
      "Train Epoch: 28 [30000/60000 (50%)]\tLoss: 137.882012\n",
      "Train Epoch: 28 [40000/60000 (67%)]\tLoss: 138.434531\n",
      "Train Epoch: 28 [50000/60000 (83%)]\tLoss: 138.943125\n",
      "====> Epoch: 28 Average loss: 137.1461\n",
      "====> Test set loss: 140.0209\n",
      "Train Epoch: 29 [0/60000 (0%)]\tLoss: 140.927969\n",
      "Train Epoch: 29 [10000/60000 (17%)]\tLoss: 140.535137\n",
      "Train Epoch: 29 [20000/60000 (33%)]\tLoss: 141.776250\n",
      "Train Epoch: 29 [30000/60000 (50%)]\tLoss: 133.684404\n",
      "Train Epoch: 29 [40000/60000 (67%)]\tLoss: 134.429873\n",
      "Train Epoch: 29 [50000/60000 (83%)]\tLoss: 142.776914\n",
      "====> Epoch: 29 Average loss: 137.2176\n",
      "====> Test set loss: 139.2913\n",
      "Train Epoch: 30 [0/60000 (0%)]\tLoss: 134.787383\n",
      "Train Epoch: 30 [10000/60000 (17%)]\tLoss: 135.347168\n",
      "Train Epoch: 30 [20000/60000 (33%)]\tLoss: 129.021172\n",
      "Train Epoch: 30 [30000/60000 (50%)]\tLoss: 135.421777\n",
      "Train Epoch: 30 [40000/60000 (67%)]\tLoss: 145.976670\n",
      "Train Epoch: 30 [50000/60000 (83%)]\tLoss: 137.861523\n",
      "====> Epoch: 30 Average loss: 137.1490\n",
      "====> Test set loss: 139.6758\n",
      "Train Epoch: 31 [0/60000 (0%)]\tLoss: 141.123486\n",
      "Train Epoch: 31 [10000/60000 (17%)]\tLoss: 135.101309\n",
      "Train Epoch: 31 [20000/60000 (33%)]\tLoss: 138.766084\n",
      "Train Epoch: 31 [30000/60000 (50%)]\tLoss: 127.551387\n",
      "Train Epoch: 31 [40000/60000 (67%)]\tLoss: 135.764375\n",
      "Train Epoch: 31 [50000/60000 (83%)]\tLoss: 131.032520\n",
      "====> Epoch: 31 Average loss: 136.8402\n",
      "====> Test set loss: 139.7194\n",
      "Train Epoch: 32 [0/60000 (0%)]\tLoss: 142.736982\n",
      "Train Epoch: 32 [10000/60000 (17%)]\tLoss: 139.045371\n",
      "Train Epoch: 32 [20000/60000 (33%)]\tLoss: 134.651514\n",
      "Train Epoch: 32 [30000/60000 (50%)]\tLoss: 136.876396\n",
      "Train Epoch: 32 [40000/60000 (67%)]\tLoss: 135.346504\n",
      "Train Epoch: 32 [50000/60000 (83%)]\tLoss: 134.621885\n",
      "====> Epoch: 32 Average loss: 136.5707\n",
      "====> Test set loss: 139.3775\n",
      "Train Epoch: 33 [0/60000 (0%)]\tLoss: 141.076514\n",
      "Train Epoch: 33 [10000/60000 (17%)]\tLoss: 137.006504\n",
      "Train Epoch: 33 [20000/60000 (33%)]\tLoss: 129.469180\n",
      "Train Epoch: 33 [30000/60000 (50%)]\tLoss: 136.468574\n",
      "Train Epoch: 33 [40000/60000 (67%)]\tLoss: 133.814434\n",
      "Train Epoch: 33 [50000/60000 (83%)]\tLoss: 131.988301\n",
      "====> Epoch: 33 Average loss: 136.8771\n",
      "====> Test set loss: 139.2557\n",
      "Train Epoch: 34 [0/60000 (0%)]\tLoss: 135.437598\n",
      "Train Epoch: 34 [10000/60000 (17%)]\tLoss: 139.194639\n",
      "Train Epoch: 34 [20000/60000 (33%)]\tLoss: 132.789697\n",
      "Train Epoch: 34 [30000/60000 (50%)]\tLoss: 132.633711\n",
      "Train Epoch: 34 [40000/60000 (67%)]\tLoss: 142.541729\n",
      "Train Epoch: 34 [50000/60000 (83%)]\tLoss: 134.068262\n",
      "====> Epoch: 34 Average loss: 136.5792\n",
      "====> Test set loss: 139.4108\n",
      "Train Epoch: 35 [0/60000 (0%)]\tLoss: 135.583926\n",
      "Train Epoch: 35 [10000/60000 (17%)]\tLoss: 140.081631\n",
      "Train Epoch: 35 [20000/60000 (33%)]\tLoss: 141.877041\n",
      "Train Epoch: 35 [30000/60000 (50%)]\tLoss: 138.790859\n",
      "Train Epoch: 35 [40000/60000 (67%)]\tLoss: 135.692295\n",
      "Train Epoch: 35 [50000/60000 (83%)]\tLoss: 143.848242\n",
      "====> Epoch: 35 Average loss: 136.2225\n",
      "====> Test set loss: 138.9179\n",
      "Train Epoch: 36 [0/60000 (0%)]\tLoss: 131.719736\n",
      "Train Epoch: 36 [10000/60000 (17%)]\tLoss: 146.343125\n",
      "Train Epoch: 36 [20000/60000 (33%)]\tLoss: 130.940361\n",
      "Train Epoch: 36 [30000/60000 (50%)]\tLoss: 144.410430\n",
      "Train Epoch: 36 [40000/60000 (67%)]\tLoss: 134.612510\n",
      "Train Epoch: 36 [50000/60000 (83%)]\tLoss: 137.964111\n",
      "====> Epoch: 36 Average loss: 136.2020\n",
      "====> Test set loss: 139.1558\n",
      "Train Epoch: 37 [0/60000 (0%)]\tLoss: 137.578770\n",
      "Train Epoch: 37 [10000/60000 (17%)]\tLoss: 133.765059\n",
      "Train Epoch: 37 [20000/60000 (33%)]\tLoss: 127.081289\n",
      "Train Epoch: 37 [30000/60000 (50%)]\tLoss: 131.412734\n",
      "Train Epoch: 37 [40000/60000 (67%)]\tLoss: 144.549922\n",
      "Train Epoch: 37 [50000/60000 (83%)]\tLoss: 134.609033\n",
      "====> Epoch: 37 Average loss: 136.4747\n",
      "====> Test set loss: 139.2521\n",
      "Train Epoch: 38 [0/60000 (0%)]\tLoss: 138.307549\n",
      "Train Epoch: 38 [10000/60000 (17%)]\tLoss: 130.883496\n",
      "Train Epoch: 38 [20000/60000 (33%)]\tLoss: 127.738096\n",
      "Train Epoch: 38 [30000/60000 (50%)]\tLoss: 133.381318\n",
      "Train Epoch: 38 [40000/60000 (67%)]\tLoss: 131.690869\n",
      "Train Epoch: 38 [50000/60000 (83%)]\tLoss: 132.273027\n",
      "====> Epoch: 38 Average loss: 136.0369\n",
      "====> Test set loss: 139.5859\n",
      "Train Epoch: 39 [0/60000 (0%)]\tLoss: 139.322285\n",
      "Train Epoch: 39 [10000/60000 (17%)]\tLoss: 137.141777\n",
      "Train Epoch: 39 [20000/60000 (33%)]\tLoss: 140.075361\n",
      "Train Epoch: 39 [30000/60000 (50%)]\tLoss: 140.404668\n",
      "Train Epoch: 39 [40000/60000 (67%)]\tLoss: 128.676797\n",
      "Train Epoch: 39 [50000/60000 (83%)]\tLoss: 139.578506\n",
      "====> Epoch: 39 Average loss: 135.8051\n",
      "====> Test set loss: 138.9922\n",
      "Train Epoch: 40 [0/60000 (0%)]\tLoss: 141.986875\n",
      "Train Epoch: 40 [10000/60000 (17%)]\tLoss: 138.473252\n",
      "Train Epoch: 40 [20000/60000 (33%)]\tLoss: 136.495098\n",
      "Train Epoch: 40 [30000/60000 (50%)]\tLoss: 139.934902\n",
      "Train Epoch: 40 [40000/60000 (67%)]\tLoss: 142.396201\n",
      "Train Epoch: 40 [50000/60000 (83%)]\tLoss: 137.354639\n",
      "====> Epoch: 40 Average loss: 135.7014\n",
      "====> Test set loss: 138.6938\n",
      "Train Epoch: 41 [0/60000 (0%)]\tLoss: 139.626318\n",
      "Train Epoch: 41 [10000/60000 (17%)]\tLoss: 134.918281\n",
      "Train Epoch: 41 [20000/60000 (33%)]\tLoss: 136.498047\n",
      "Train Epoch: 41 [30000/60000 (50%)]\tLoss: 134.687285\n",
      "Train Epoch: 41 [40000/60000 (67%)]\tLoss: 140.884863\n",
      "Train Epoch: 41 [50000/60000 (83%)]\tLoss: 138.896504\n",
      "====> Epoch: 41 Average loss: 135.7786\n",
      "====> Test set loss: 138.8732\n",
      "Train Epoch: 42 [0/60000 (0%)]\tLoss: 136.115557\n",
      "Train Epoch: 42 [10000/60000 (17%)]\tLoss: 128.833652\n",
      "Train Epoch: 42 [20000/60000 (33%)]\tLoss: 139.057168\n",
      "Train Epoch: 42 [30000/60000 (50%)]\tLoss: 139.534189\n",
      "Train Epoch: 42 [40000/60000 (67%)]\tLoss: 145.967451\n",
      "Train Epoch: 42 [50000/60000 (83%)]\tLoss: 142.957559\n",
      "====> Epoch: 42 Average loss: 135.4694\n",
      "====> Test set loss: 138.7231\n",
      "Train Epoch: 43 [0/60000 (0%)]\tLoss: 124.132109\n",
      "Train Epoch: 43 [10000/60000 (17%)]\tLoss: 135.753711\n",
      "Train Epoch: 43 [20000/60000 (33%)]\tLoss: 141.724102\n",
      "Train Epoch: 43 [30000/60000 (50%)]\tLoss: 136.230312\n",
      "Train Epoch: 43 [40000/60000 (67%)]\tLoss: 134.490117\n",
      "Train Epoch: 43 [50000/60000 (83%)]\tLoss: 143.411162\n",
      "====> Epoch: 43 Average loss: 135.4557\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====> Test set loss: 138.6256\n",
      "Train Epoch: 44 [0/60000 (0%)]\tLoss: 135.063555\n",
      "Train Epoch: 44 [10000/60000 (17%)]\tLoss: 133.560957\n",
      "Train Epoch: 44 [20000/60000 (33%)]\tLoss: 136.025967\n",
      "Train Epoch: 44 [30000/60000 (50%)]\tLoss: 131.632178\n",
      "Train Epoch: 44 [40000/60000 (67%)]\tLoss: 135.976533\n",
      "Train Epoch: 44 [50000/60000 (83%)]\tLoss: 144.262275\n",
      "====> Epoch: 44 Average loss: 135.4606\n",
      "====> Test set loss: 139.1682\n",
      "Train Epoch: 45 [0/60000 (0%)]\tLoss: 147.984717\n",
      "Train Epoch: 45 [10000/60000 (17%)]\tLoss: 139.047139\n",
      "Train Epoch: 45 [20000/60000 (33%)]\tLoss: 130.762754\n",
      "Train Epoch: 45 [30000/60000 (50%)]\tLoss: 136.663232\n",
      "Train Epoch: 45 [40000/60000 (67%)]\tLoss: 132.488057\n",
      "Train Epoch: 45 [50000/60000 (83%)]\tLoss: 131.014756\n",
      "====> Epoch: 45 Average loss: 135.2890\n",
      "====> Test set loss: 138.3633\n",
      "Train Epoch: 46 [0/60000 (0%)]\tLoss: 130.941514\n",
      "Train Epoch: 46 [10000/60000 (17%)]\tLoss: 145.028047\n",
      "Train Epoch: 46 [20000/60000 (33%)]\tLoss: 134.676533\n",
      "Train Epoch: 46 [30000/60000 (50%)]\tLoss: 138.489463\n",
      "Train Epoch: 46 [40000/60000 (67%)]\tLoss: 136.849863\n",
      "Train Epoch: 46 [50000/60000 (83%)]\tLoss: 136.730283\n",
      "====> Epoch: 46 Average loss: 134.9870\n",
      "====> Test set loss: 138.6371\n",
      "Train Epoch: 47 [0/60000 (0%)]\tLoss: 139.129336\n",
      "Train Epoch: 47 [10000/60000 (17%)]\tLoss: 136.708848\n",
      "Train Epoch: 47 [20000/60000 (33%)]\tLoss: 136.906631\n",
      "Train Epoch: 47 [30000/60000 (50%)]\tLoss: 137.643955\n",
      "Train Epoch: 47 [40000/60000 (67%)]\tLoss: 141.815928\n",
      "Train Epoch: 47 [50000/60000 (83%)]\tLoss: 137.937598\n",
      "====> Epoch: 47 Average loss: 134.8983\n",
      "====> Test set loss: 138.6167\n",
      "Train Epoch: 48 [0/60000 (0%)]\tLoss: 137.293193\n",
      "Train Epoch: 48 [10000/60000 (17%)]\tLoss: 135.439121\n",
      "Train Epoch: 48 [20000/60000 (33%)]\tLoss: 133.545654\n",
      "Train Epoch: 48 [30000/60000 (50%)]\tLoss: 127.829941\n",
      "Train Epoch: 48 [40000/60000 (67%)]\tLoss: 132.087187\n",
      "Train Epoch: 48 [50000/60000 (83%)]\tLoss: 133.079697\n",
      "====> Epoch: 48 Average loss: 134.8169\n",
      "====> Test set loss: 138.3705\n",
      "Train Epoch: 49 [0/60000 (0%)]\tLoss: 139.804346\n",
      "Train Epoch: 49 [10000/60000 (17%)]\tLoss: 140.827578\n",
      "Train Epoch: 49 [20000/60000 (33%)]\tLoss: 133.810625\n",
      "Train Epoch: 49 [30000/60000 (50%)]\tLoss: 134.355352\n",
      "Train Epoch: 49 [40000/60000 (67%)]\tLoss: 139.340664\n",
      "Train Epoch: 49 [50000/60000 (83%)]\tLoss: 134.977061\n",
      "====> Epoch: 49 Average loss: 134.6185\n",
      "====> Test set loss: 138.2620\n",
      "Train Epoch: 50 [0/60000 (0%)]\tLoss: 137.014463\n",
      "Train Epoch: 50 [10000/60000 (17%)]\tLoss: 134.228213\n",
      "Train Epoch: 50 [20000/60000 (33%)]\tLoss: 134.444277\n",
      "Train Epoch: 50 [30000/60000 (50%)]\tLoss: 134.437158\n",
      "Train Epoch: 50 [40000/60000 (67%)]\tLoss: 138.614404\n",
      "Train Epoch: 50 [50000/60000 (83%)]\tLoss: 139.861016\n",
      "====> Epoch: 50 Average loss: 134.6723\n",
      "====> Test set loss: 138.4024\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(1, 51):\n",
    "    train(epoch)\n",
    "    test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lyeoni\\AppData\\Local\\Continuum\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\torch\\nn\\functional.py:1006: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\n",
      "  warnings.warn(\"nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\")\n"
     ]
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    z = torch.randn(64, 2).cuda()\n",
    "    sample = vae.decoder(z).cuda()\n",
    "    \n",
    "    save_image(sample.view(64, 1, 28, 28), './samples/sample_' + '.png')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}