part1/workshop.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "155693f2",
   "metadata": {},
   "source": [
    "# ML Workshop - Part 1: MNIST"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad4434a4",
   "metadata": {},
   "source": [
    "### install packages"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d695aed5",
   "metadata": {},
   "source": [
    "restart kernel afterwards to reload packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "41553541",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install --quiet torch torchvision pytorch-lightning wandb matplotlib"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "013d04d9",
   "metadata": {},
   "source": [
    "load pytorch and test if pip install worked"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "86c0d9cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "479deea2",
   "metadata": {},
   "source": [
    "test if CUDA is available  \n",
    "**part1 works without a GPU**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7a6707fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8299505",
   "metadata": {},
   "source": [
    "### MNIST"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1269e201",
   "metadata": {},
   "source": [
    "download MNIST dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "654b561d",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset MNIST\n",
       "    Number of datapoints: 10000\n",
       "    Root location: .\n",
       "    Split: Test"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from torchvision.datasets import MNIST\n",
    "MNIST(\".\", train=True, download=True)\n",
    "MNIST(\".\", train=False, download=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a951de7c",
   "metadata": {},
   "source": [
    "more about MNIST:\n",
    "- http://yann.lecun.com/exdb/mnist/\n",
    "- https://pytorch.org/vision/stable/datasets.html#mnist"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db89a6f8",
   "metadata": {},
   "source": [
    "set seed for random. so all runs have the same random!<br/>\n",
    "https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.html?#seed-everything"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "def5b78d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 42\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "42"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pytorch_lightning as lightning\n",
    "lightning.seed_everything(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9923701a",
   "metadata": {},
   "source": [
    "split MNIST into variables  \n",
    "(split 60000 train into train/validation)  \n",
    "(never use test data for validation!)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4a421ed4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torchvision\n",
    "mnist_train= MNIST(\".\", train=True, transform=torchvision.transforms.ToTensor())\n",
    "train_ds, validation_ds = torch.utils.data.random_split(mnist_train,[55000,5000])\n",
    "#\n",
    "test_ds = MNIST(\".\", train=False, transform=torchvision.transforms.ToTensor())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "187e4c09",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([60000, 28, 28])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist_train.data.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "54c87e18",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(55000, 5000)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_ds), len(validation_ds)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7552640d",
   "metadata": {},
   "source": [
    "plot an image from test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ee9c2fd1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAEICAYAAACZA4KlAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAPQElEQVR4nO3df4xVdXrH8c8DuFYFrWg6ISyI4I9m09jZBi1JzcZmw0pVMhITskQaatFZDZqSNKZG/1gj3bhputuQNq7OAgGUupLgFkLXBUvaov0DHJTyQxBcAlkmAyOxYQe1UeDpH/ewHWHO947n3HvPhef9SiZz73nuuffJhc+cc8/3nPs1dxeAS9+oqhsA0BqEHQiCsANBEHYgCMIOBEHYgSAIOxAEYcewzOw/zOx/zexU9vNB1T2hHMKOlMfdfWz2c2vVzaAcwg4EQdiR8ryZnTCz/zKzu6puBuUY58ZjOGb2x5Lel/S5pO9K+idJne7+q0obQ2GEHSNiZr+U9K/u/o9V94Ji2I3HSLkkq7oJFEfYcQEz+10zu9vMfsfMxpjZg5K+JemXVfeG4sZU3QDa0mWS/lbS70s6I2m/pPvd/UClXaEUPrMDQbAbDwRB2IEgCDsQBGEHgmjp0Xgz42gg0GTuPuz5EKW27GY2y8w+MLMPzeypMs8FoLkKD72Z2WhJByTNlHRU0juS5rn7+4l12LIDTdaMLfsdkj5090Pu/rmkn0nqKvF8AJqoTNgnSvr1kPtHs2VfYmbdZtZrZr0lXgtASU0/QOfuPZJ6JHbjgSqV2bL3SZo05P7Xs2UA2lCZsL8j6WYzu9HMvqbaFxxsaExbABqt8G68u582s8clbZI0WtIKd9/bsM4ANFRLr3rjMzvQfE05qQbAxYOwA0EQdiAIwg4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQRB2IAjCDgRB2IEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIwg4EQdiBIApP2YzGGTduXLLe0dGRrB89ejS3dsMNNyTXXbhwYbJ+xRVXJOv13Hbbbbm1Xbt2Jdd96aWXkvU9e/YU6imqUmE3s8OSBiWdkXTa3ac3oikAjdeILfufuvuJBjwPgCbiMzsQRNmwu6TNZrbDzLqHe4CZdZtZr5n1lnwtACWU3Y2/0937zOz3JL1pZvvdfevQB7h7j6QeSTIzL/l6AAoqtWV3977s94Ckn0u6oxFNAWi8wmE3s6vMbNy525K+I4mxEKBNmXuxPWszm6ra1lyqfRz4Z3f/QZ112I0fxsqVK5P1+fPnJ+vbt2/Prc2YMaNIS22hr68vWZ85c2ayvn///ka2c9FwdxtueeHP7O5+SNIfFu4IQEsx9AYEQdiBIAg7EARhB4Ig7EAQXOLaAmbDjoT81pQpU5L1UaPSf5PLDK998sknyfqSJUuS9bVr1ybr3d3DnkUtSZo9e3Zy3VtuuSVZnzx5crIedegtD1t2IAjCDgRB2IEgCDsQBGEHgiDsQBCEHQii8CWuhV4s6CWuTzzxRLK+dOnSFnVyoXpfU/3RRx+1qBM0St4lrmzZgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIrmdvgHrXm993332lnv/AgQPJ+t69e3NrZ86cSa578uTJQj3h4sOWHQiCsANBEHYgCMIOBEHYgSAIOxAEYQeC4Hr2Bti2bVuyfvvtt5d6/lOnTiXrqbHycePGJdcdHBws1NNILVu2LLe2fv365LoHDx5M1j/99NNkvZX/t9tJ4evZzWyFmQ2Y2Z4hy8ab2ZtmdjD7fW0jmwXQeCPZjV8padZ5y56StMXdb5a0JbsPoI3VDbu7b5X08XmLuyStym6vknR/Y9sC0GhFz43vcPf+7PYxSblfZGZm3ZLyJ/wC0BKlL4Rxd08deHP3Hkk90qV7gA64GBQdejtuZhMkKfs90LiWADRD0bBvkLQgu71AUnoMBUDl6o6zm9mrku6SdL2k45K+L+lfJK2VNFnSEUlz3f38g3jDPddFuxvf2dmZW9u6dWty3bFjxza4G0jSiy++mKynxvj37duXXPezzz4r1FM7yBtnr/uZ3d3n5ZS+XaojAC3F6bJAEIQdCIKwA0EQdiAIwg4EwSWuI/TMM8/k1pYsWdLU196wYUOynhomWrduXXLd/v7+ZL2euXPnJuuTJ0/OrXV1dZV67TJefvnlZP2hhx5K1s+ePdvIdhqKKZuB4Ag7EARhB4Ig7EAQhB0IgrADQRB2IAjG2Ufo1ltvza1t3rw5uW5vb2+yXm+cfvfu3cl6vWmZq5Sazvq6665Lrvvcc88l6/Pm5V2QWXPNNdfk1ur9v1+9enWy/sgjjyTrX3zxRbLeTIyzA8ERdiAIwg4EQdiBIAg7EARhB4Ig7EAQjLPjonXllVcm66+99lpu7d577y312vXG2ZcvX17q+ctgnB0IjrADQRB2IAjCDgRB2IEgCDsQBGEHgmCcHZesq6++Orf23nvvJde98cYbk/V632Ewa9asZL2ZCo+zm9kKMxswsz1Dlj1rZn1mtjP7uaeRzQJovJHsxq+UNNyfqX9w987s5xeNbQtAo9UNu7tvlfRxC3oB0ERlDtA9bma7st38a/MeZGbdZtZrZukvYgPQVEXD/hNJ0yR1SuqX9KO8B7p7j7tPd/fpBV8LQAMUCru7H3f3M+5+VtJPJd3R2LYANFqhsJvZhCF350jak/dYAO2h7ji7mb0q6S5J10s6Lun72f1OSS7psKTvuXvdib4ZZ0e7mDFjRrL+9ttvJ+uHDx8u9fwnTpxI1svIG2cfM4IVh/sm/uquzAdQCKfLAkEQdiAIwg4EQdiBIAg7EETdo/Gob/r09MmBfX19yXp/f91RSzTYkSNHkvV6Q9JTp05N1sePH5+sN3PoLQ9bdiAIwg4EQdiBIAg7EARhB4Ig7EAQhB0IgnH2EXrhhRdyawsXLkyuu2zZsmR90aJFhXpCcV1dXcn66NGjW9RJ67BlB4Ig7EAQhB0IgrADQRB2IAjCDgRB2IEgmLI5c9NNNyXr+/fvz6299dZbyXVnzpyZrJ8+fTpZRzHTpk3LrdX7quiOjo5kff369cn6Aw88kKyfPXs2WS+j8JTNAC4NhB0IgrADQRB2IAjCDgRB2IEgCDsQRN3r2c1skqTVkjpUm6K5x92Xmtl4Sa9JmqLatM1z3f1/mtdqc82ePTtZHzUq/+/ijh07kusyjl6NRx99NLdWbxx9YGAgWX/ssceS9WaOoxc1ki37aUl/7e7fkDRD0iIz+4akpyRtcfebJW3J7gNoU3XD7u797v5udntQ0j5JEyV1SVqVPWyVpPub1COABvhKn9nNbIqkb0raJqnD3c/NW3RMtd18AG1qxN9BZ2ZjJa2TtNjdf2P2/6ffurvnnfduZt2Suss2CqCcEW3Zzewy1YK+xt1fzxYfN7MJWX2CpGGPaLh7j7tPd/f07IcAmqpu2K22CV8uaZ+7/3hIaYOkBdntBZLSlwEBqNRIduP/RNKfS9ptZjuzZU9L+qGktWa2UNIRSXOb0mGL1JuCF+1n8eLFyXp3d/FPj9u3b0/Wjx07Vvi5q1I37O7+tqRhr4+V9O3GtgOgWTiDDgiCsANBEHYgCMIOBEHYgSAIOxAEXyWdmTNnTrK+Zs2a3NqYMekRzHqXz27atClZv1R1dnYm688//3yyfvfddxd+7Xrveb1/s3a+bJmvkgaCI+xAEIQdCIKwA0EQdiAIwg4EQdiBIBhnH6Genp7c2sMPP5xc99SpU8n6ihUrkvU33ngjWd+zZ09ura+vL7luWTNmzEjWn3zyycLrTpgwoVBP52zcuDG39uCDDybXHRwcLPXaVWKcHQiOsANBEHYgCMIOBEHYgSAIOxAEYQeCYJx9hMaOHZtb27dvX3LdiRMnNrqdL0lNL3zy5Mnkuq+88kqyPn/+/GR98uTJyfrll1+erJdR73r4Q4cO5dbqnftwMWOcHQiOsANBEHYgCMIOBEHYgSAIOxAEYQeCqDvObmaTJK2W1CHJJfW4+1Ize1bSI5I+yh76tLv/os5zXbTj7MDFIm+cfSRhnyBpgru/a2bjJO2QdL+kuZJOufvfj7QJwg40X17Y01OZ1Fbsl9Sf3R40s32SmntKGICG+0qf2c1siqRvStqWLXrczHaZ2QozuzZnnW4z6zWz3nKtAihjxOfGm9lYSf8p6Qfu/rqZdUg6odrn+CWq7er/ZZ3nYDceaLLCn9klycwuk7RR0iZ3//Ew9SmSNrr7H9R5HsIONFnhC2HMzCQtl7RvaNCzA3fnzJGU/xWnACo3kqPxd0p6S9JuSWezxU9LmiepU7Xd+MOSvpcdzEs9F1t2oMlK7cY3CmEHmo/r2YHgCDsQBGEHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSAIOxAEYQeCIOxAEIQdCIKwA0HU/cLJBjsh6ciQ+9dny9pRu/bWrn1J9FZUI3u7Ia/Q0uvZL3hxs153n15ZAwnt2lu79iXRW1Gt6o3deCAIwg4EUXXYeyp+/ZR27a1d+5LoraiW9FbpZ3YArVP1lh1AixB2IIhKwm5ms8zsAzP70MyeqqKHPGZ22Mx2m9nOqueny+bQGzCzPUOWjTezN83sYPZ72Dn2KurtWTPry967nWZ2T0W9TTKzfzez981sr5n9Vba80vcu0VdL3reWf2Y3s9GSDkiaKemopHckzXP391vaSA4zOyxpurtXfgKGmX1L0ilJq89NrWVmfyfpY3f/YfaH8lp3/5s26e1ZfcVpvJvUW94043+hCt+7Rk5/XkQVW/Y7JH3o7ofc/XNJP5PUVUEfbc/dt0r6+LzFXZJWZbdXqfafpeVyemsL7t7v7u9mtwclnZtmvNL3LtFXS1QR9omSfj3k/lG113zvLmmzme0ws+6qmxlGx5Bpto5J6qiymWHUnca7lc6bZrxt3rsi05+XxQG6C93p7n8k6c8kLcp2V9uS1z6DtdPY6U8kTVNtDsB+ST+qsplsmvF1kha7+2+G1qp874bpqyXvWxVh75M0acj9r2fL2oK792W/ByT9XLWPHe3k+LkZdLPfAxX381vuftzdz7j7WUk/VYXvXTbN+DpJa9z99Wxx5e/dcH216n2rIuzvSLrZzG40s69J+q6kDRX0cQEzuyo7cCIzu0rSd9R+U1FvkLQgu71A0voKe/mSdpnGO2+acVX83lU+/bm7t/xH0j2qHZH/laRnqughp6+pkv47+9lbdW+SXlVtt+4L1Y5tLJR0naQtkg5K+jdJ49uot5dVm9p7l2rBmlBRb3eqtou+S9LO7Oeeqt+7RF8ted84XRYIggN0QBCEHQiCsANBEHYgCMIOBEHYgSAIOxDE/wGCIvdgRbk7OgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "number = random.randint(0, 1000)\n",
    "plt.imshow(test_ds.data[number], cmap='gray')\n",
    "plt.title('%i' % test_ds.targets[number])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50b922d6",
   "metadata": {},
   "source": [
    "show first dataset as tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "58ca73a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# test_ds[0][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea42bcbd",
   "metadata": {},
   "source": [
    "class of first test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e35062a9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(7)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_ds.targets[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98603af0",
   "metadata": {},
   "source": [
    "### weights & biases"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05d47d83",
   "metadata": {},
   "source": [
    "get your token here: https://wandb.ai/settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a0000f82",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mmfa\u001b[0m (use `wandb login --relogin` to force relogin)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import wandb\n",
    "wandb.login()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30b39be1",
   "metadata": {},
   "source": [
    "### The model!"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd2ce69c",
   "metadata": {},
   "source": [
    "![](https://miro.medium.com/max/700/1*HWhBextdDSkxYvz0kEMTVg.png)<br/>\n",
    "(image source: https://towardsdatascience.com/handwritten-digit-mnist-pytorch-977b5338e627)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63119662",
   "metadata": {},
   "source": [
    "CrossEntropyLoss(x, y) := NLLLoss(LogSoftmax(x), y) -- https://pytorch.org/docs/stable/generated/torch.nn.CrossEntropyLoss.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "cbb8813d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchmetrics import Accuracy\n",
    "from torch.nn import functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a73b090c",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MNISTModel(lightning.LightningModule):\n",
    "    def __init__(self, hidden_size=10, learning_rate=1e-2):\n",
    "        super().__init__()\n",
    "        self.save_hyperparameters()\n",
    "\n",
    "        self.learning_rate = learning_rate\n",
    "        self.batch_size = 128\n",
    "\n",
    "        # 10 different numers; images are 28x28 greyscale\n",
    "        self.num_classes = 10\n",
    "        channels, width, height = (1, 28, 28)\n",
    "\n",
    "        # simple NN\n",
    "        self.model = torch.nn.Sequential(\n",
    "            torch.nn.Flatten(),\n",
    "            torch.nn.Linear(channels * width * height, hidden_size),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(hidden_size, self.num_classes),\n",
    "        )\n",
    "        self.loss = torch.nn.CrossEntropyLoss()\n",
    "\n",
    "        acc = Accuracy()\n",
    "        self.train_acc = acc.clone()\n",
    "        self.valid_acc = acc.clone()\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.model(x)\n",
    "\n",
    "    def training_step(self, batch, batch_nb):\n",
    "        x, y = batch\n",
    "        y_hat = self.forward(x)\n",
    "        loss = self.loss(y_hat, y)\n",
    "        self.log(\"train/loss\", loss)\n",
    "\n",
    "        logits = self(x)\n",
    "        preds = torch.argmax(logits, dim=1)\n",
    "        self.train_acc(preds, y)\n",
    "        self.log(\"train/acc\", self.train_acc)\n",
    "        return {\"loss\": loss}\n",
    "\n",
    "    def validation_step(self, batch, batch_idx, name=\"val\"):\n",
    "        x, y = batch\n",
    "        y_hat = self.forward(x)\n",
    "        loss = self.loss(y_hat, y)\n",
    "        self.log(f\"{name}/loss\", loss)\n",
    "\n",
    "        logits = self(x)\n",
    "        preds = torch.argmax(logits, dim=1)\n",
    "        self.valid_acc(preds, y)\n",
    "\n",
    "        self.log(f\"{name}/acc\", self.valid_acc)\n",
    "        return {\"loss\": loss}\n",
    "\n",
    "    def test_step(self, batch, batch_idx):\n",
    "        return self.validation_step(batch, batch_idx, name=\"test\")\n",
    "\n",
    "    def configure_optimizers(self):\n",
    "        return torch.optim.Adam(self.parameters(), lr=self.learning_rate)\n",
    "\n",
    "    def train_dataloader(self):\n",
    "        return torch.utils.data.DataLoader(train_ds, batch_size=self.batch_size)\n",
    "\n",
    "    def val_dataloader(self):\n",
    "        return torch.utils.data.DataLoader(validation_ds, batch_size=self.batch_size)\n",
    "\n",
    "    def test_dataloader(self):\n",
    "        return torch.utils.data.DataLoader(test_ds, batch_size=self.batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "503f89f7",
   "metadata": {},
   "source": [
    "links for questions:\n",
    "- activation function: https://pytorch.org/docs/stable/generated/torch.nn.ReLU.html\n",
    "- optimizer: https://pytorch.org/docs/stable/generated/torch.optim.Adam.html"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4fa2884b",
   "metadata": {},
   "source": [
    "### train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d8f8258c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "                    Syncing run <strong><a href=\"https://wandb.ai/mfa/ml-workshop-part1--MNIST/runs/19ahjxjk\" target=\"_blank\">lilac-mountain-11</a></strong> to <a href=\"https://wandb.ai/mfa/ml-workshop-part1--MNIST\" target=\"_blank\">Weights & Biases</a> (<a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">docs</a>).<br/>\n",
       "\n",
       "                "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: logging graph, to disable use `wandb.watch(log_graph=False)`\n",
      "GPU available: True, used: False\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "/home/mfa/.virtualenvs/machine-learning-workshop/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py:1584: UserWarning: GPU available but not used. Set the gpus flag in your trainer `Trainer(gpus=1)` or script `--gpus=1`.\n",
      "  rank_zero_warn(\n",
      "\n",
      "  | Name      | Type             | Params\n",
      "-----------------------------------------------\n",
      "0 | model     | Sequential       | 8.0 K \n",
      "1 | loss      | CrossEntropyLoss | 0     \n",
      "2 | train_acc | Accuracy         | 0     \n",
      "3 | valid_acc | Accuracy         | 0     \n",
      "-----------------------------------------------\n",
      "8.0 K     Trainable params\n",
      "0         Non-trainable params\n",
      "8.0 K     Total params\n",
      "0.032     Total estimated model params size (MB)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation sanity check: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mfa/.virtualenvs/machine-learning-workshop/lib/python3.9/site-packages/pytorch_lightning/trainer/data_loading.py:132: UserWarning: The dataloader, val_dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 8 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  rank_zero_warn(\n",
      "Global seed set to 42\n",
      "/home/mfa/.virtualenvs/machine-learning-workshop/lib/python3.9/site-packages/pytorch_lightning/trainer/data_loading.py:132: UserWarning: The dataloader, train_dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 8 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  rank_zero_warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8bbfac201da842caa0711ead20a5a085",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = MNISTModel()\n",
    "wandb_logger = lightning.loggers.WandbLogger(project=\"ml-workshop-part1--MNIST\")\n",
    "wandb_logger.watch(model, log=\"all\")\n",
    "trainer = lightning.Trainer(gpus=0, max_epochs=3, logger=wandb_logger)\n",
    "trainer.fit(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1b13331a",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Restoring states from the checkpoint path at /home/mfa/machine-learning-workshop/ml-workshop-part1--MNIST/19ahjxjk/checkpoints/epoch=2-step=1289.ckpt\n",
      "Loaded model weights from checkpoint at /home/mfa/machine-learning-workshop/ml-workshop-part1--MNIST/19ahjxjk/checkpoints/epoch=2-step=1289.ckpt\n",
      "/home/mfa/.virtualenvs/machine-learning-workshop/lib/python3.9/site-packages/pytorch_lightning/trainer/data_loading.py:132: UserWarning: The dataloader, test_dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 8 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  rank_zero_warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8028a75fa5a743ea8e769118e32c3e25",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Testing: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------------------------------------\n",
      "DATALOADER:0 TEST RESULTS\n",
      "{'test/acc': 0.9261999726295471, 'test/loss': 0.2697216868400574}\n",
      "--------------------------------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'test/loss': 0.2697216868400574, 'test/acc': 0.9261999726295471}]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.test(ckpt_path='best')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1a379241",
   "metadata": {},
   "source": [
    "finish wandb run. otherwise the next training is appended!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ad4be6fe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<br/>Waiting for W&B process to finish, PID 607956... <strong style=\"color:green\">(success).</strong>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(Label(value=' 0.00MB of 0.00MB uploaded (0.00MB deduped)\\r'), FloatProgress(value=1.0, max=1.0)…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: right }\n",
       "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; width: 100% }\n",
       "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
       "    </style>\n",
       "<div class=\"wandb-row\"><div class=\"wandb-col\">\n",
       "<h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>epoch</td><td>▁▁▁▁▁▁▁▁▁▃▃▃▃▃▃▃▃▃▃▆▆▆▆▆▆▆▆▆█</td></tr><tr><td>test/acc</td><td>▁</td></tr><tr><td>test/loss</td><td>▁</td></tr><tr><td>train/acc</td><td>▁▁▄▄▂▇▅▅▁▇▅▅▇▃▄▅▅▄▅▅█▃▅▇▇</td></tr><tr><td>train/loss</td><td>▆█▃▅▅▂▄▃▄▂▅▅▃▄▃▄▂▃▃▄▁▃▂▂▁</td></tr><tr><td>trainer/global_step</td><td>▁▁▂▂▂▂▃▃▃▃▄▄▄▄▅▅▅▆▆▆▆▆▇▇▇▇███</td></tr><tr><td>val/acc</td><td>▁██</td></tr><tr><td>val/loss</td><td>█▁▁</td></tr></table><br/></div><div class=\"wandb-col\">\n",
       "<h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>epoch</td><td>3</td></tr><tr><td>test/acc</td><td>0.9262</td></tr><tr><td>test/loss</td><td>0.26972</td></tr><tr><td>train/acc</td><td>0.95312</td></tr><tr><td>train/loss</td><td>0.17387</td></tr><tr><td>trainer/global_step</td><td>1290</td></tr><tr><td>val/acc</td><td>0.919</td></tr><tr><td>val/loss</td><td>0.28772</td></tr></table>\n",
       "</div></div>\n",
       "Synced 5 W&B file(s), 1 media file(s), 0 artifact file(s) and 0 other file(s)\n",
       "<br/>Synced <strong style=\"color:#cdcd00\">lilac-mountain-11</strong>: <a href=\"https://wandb.ai/mfa/ml-workshop-part1--MNIST/runs/19ahjxjk\" target=\"_blank\">https://wandb.ai/mfa/ml-workshop-part1--MNIST/runs/19ahjxjk</a><br/>\n",
       "Find logs at: <code>./wandb/run-20220223_145307-19ahjxjk/logs</code><br/>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "wandb.finish()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "685618fa",
   "metadata": {},
   "source": [
    "### show one test example classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "37f81e80",
   "metadata": {},
   "outputs": [],
   "source": [
    "image_number = 23"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "321ee61b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(5)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_ds.targets[image_number]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a737896",
   "metadata": {},
   "source": [
    "the image needs to become a one element batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "56e2ab47",
   "metadata": {},
   "outputs": [],
   "source": [
    "one_dataset = test_ds[image_number][0].reshape(1, 1, 28, 28)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e7d91c53",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 1, 28, 28])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "one_dataset.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "cf5773d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ -1.4970,  -8.2735,  -5.4512,   1.2322,   0.2442,   8.0919,   3.2754,\n",
       "         -11.2216,   4.1386,  -4.7485]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.model(one_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "c01c7007",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([5])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.argmax(trainer.model(one_dataset), dim=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dbe27e9a",
   "metadata": {},
   "source": [
    "target is **5** and the model says **5**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e79c0bb8",
   "metadata": {},
   "source": [
    "### try an image we created ourselves\n",
    "\n",
    "example image: https://paste.madflex.de/F6CYd96D/+inline \n",
    "the image should be 28 x 28, 1-bit colormap, with a 4 pixel border around"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "867acb1f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SSL_INIT\r\n"
     ]
    }
   ],
   "source": [
    "!wget https://paste.madflex.de/F6CYd96D/+inline -q -O own.png"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "5d1b1896",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<PIL.Image.Image image mode=L size=28x28 at 0x7F987BBD05E0>\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAPkUlEQVR4nO3df4xV9ZnH8c8DC6LMxIDGCVJYoEgMGksRwUTYdDVtXBID/cMKUaNZ0mlC0RpNdomrQNysMbtbV/0HpSkpbKhIgqYEi4UljfgjEhFRESy4OAo4zoAQC4mKyLN/zGEz6pzvGe65v+B5v5LJ3DnPnHser/PhnHu/55yvubsAnPsGNLoBAPVB2IEgCDsQBGEHgiDsQBB/U8+NmRkf/QM15u7W1/JSe3Yzu9HM/mJm75vZwjLPBaC2rNJxdjMbKGmPpB9LOiDpdUlz3X1XYh327ECN1WLPPlXS++6+z91PSFotaVaJ5wNQQ2XCPlLS/l4/H8iWfYOZtZvZNjPbVmJbAEqq+Qd07r5M0jKJw3igkcrs2Q9KGtXr5+9lywA0oTJhf13SZWY21swGS5ojaV112gJQbRUfxrv7STNbIOlPkgZKWu7u71atMwBVVfHQW0Ub4z07UHM1OakGwNmDsANBEHYgCMIOBEHYgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQRB2IAjCDgRB2IEg6jplM2qjpaUlt3b77bcn1502bVqyftFFFyXrhw8fTtZXrlyZW3vzzTeT61555ZXJ+jXXXJOsP/PMM7m1jz/+OLnuuYg9OxAEYQeCIOxAEIQdCIKwA0EQdiAIwg4EwSyuZ4E777wzWb/vvvtya5dffnly3YEDB1bSUr/t2bMnt/boo48m17333nuT9bFjxybrL7zwQm7t1ltvTa57/PjxZL2Z5c3iWuqkGjPrkHRM0teSTrr7lDLPB6B2qnEG3d+7e/o0KgANx3t2IIiyYXdJG83sDTNr7+sXzKzdzLaZ2baS2wJQQtnD+OnuftDMLpG0yczec/ctvX/B3ZdJWibxAR3QSKX27O5+MPveLek5SVOr0RSA6qs47GY21MxaTz+W9BNJO6vVGIDqKnMY3ybpOTM7/Ty/d/f8gc3ABgxI/5u6aNGiZH3hwoXJ+nnnnXfGPVXLqVOnKq6PGzcuuW5RfdCgQcn6zJkzc2uTJk1Krvvyyy8n62ejisPu7vsk/aCKvQCoIYbegCAIOxAEYQeCIOxAEIQdCIJbSdfB5MmTk/XUJapSuaG1oqGxQ4cOJesdHR3J+u7du5P11K2kiy6vnj9/frJeNPSWGvKcMWNGct1zceiNPTsQBGEHgiDsQBCEHQiCsANBEHYgCMIOBME4exUsWbIkWV+8eHGp5z969Giyfu211+bWUrdyrofx48fn1jZs2JBct7W1tdS2U+cYvPbaa6We+2zEnh0IgrADQRB2IAjCDgRB2IEgCDsQBGEHgmCcvQpGjx6drBddt53djjvXl19+WWr9WrrhhhuS9aeeeiq3VnSr6CJF1+pv2rQpt/bSSy+V2vbZiD07EARhB4Ig7EAQhB0IgrADQRB2IAjCDgTBOHsV7Nu3L1kvO85+/vnnJ+tlrvseMmRIsn733Xcn6w8++GCy3tLScsY99ddbb72VrM+bNy+3dvLkyWq30/QK9+xmttzMus1sZ69lw81sk5ntzb4Pq22bAMrqz2H87yTd+K1lCyVtdvfLJG3OfgbQxArD7u5bJB351uJZklZkj1dIml3dtgBUW6Xv2dvcvTN7/ImktrxfNLN2Se0VbgdAlZT+gM7d3cxyP4Fy92WSlklS6vcA1FalQ29dZjZCkrLv3dVrCUAtVBr2dZLuyB7fIekP1WkHQK0UHsab2dOSfiTpYjM7IGmxpEckrTGzeZI+lPSzWjbZ7Pbu3ZusF43pDh48OFkvGkefM2dObu2LL75Irrto0aJkffbs2cl60RzpKUXnH7z33nvJ+m233Zasd3Z2JuvRFIbd3efmlNJ3LQDQVDhdFgiCsANBEHYgCMIOBEHYgSCsaPijqhs7R8+ga2vLPVtYkrRx48Zk/aqrriq1/dTQXtHtlouG/co6ceJEbm39+vXJdefPn5+sd3V1VdTTuc7d+7xmmj07EARhB4Ig7EAQhB0IgrADQRB2IAjCDgTBOHsdXHfddcl6amphqfhW0rVU9Pfx2WefJesPP/xwbu2JJ55Irls0VTX6xjg7EBxhB4Ig7EAQhB0IgrADQRB2IAjCDgTBlM11MGHChEa3ULGicfai22jv2LEjt3bBBRck1/3qq6+S9aJr9fFN7NmBIAg7EARhB4Ig7EAQhB0IgrADQRB2IAiuZ6+CoqmDly5dmqy3tLRUs526Kvr7SV2TnhqDl6R169Yl688//3yyvmvXrtxa0TTaZ7OKr2c3s+Vm1m1mO3stW2JmB81sR/Y1s5rNAqi+/hzG/07SjX0s/y93n5R9/bG6bQGotsKwu/sWSUfq0AuAGirzAd0CM3s7O8wflvdLZtZuZtvMbFuJbQEoqdKwL5X0fUmTJHVK+nXeL7r7Mnef4u5TKtwWgCqoKOzu3uXuX7v7KUm/kTS1um0BqLaKwm5mI3r9+FNJO/N+F0BzKBxnN7OnJf1I0sWSuiQtzn6eJMkldUj6hbt3Fm7sLB5nT81jvmHDhuS6119/faltF/0/Wrt2bW7twgsvTK47ffr0ZH3IkCHJulmfQ7pVUfTf/fnnnyfrr776am5t3rx5yXU/+uijZL2Z5Y2zF968wt3n9rH4t6U7AlBXnC4LBEHYgSAIOxAEYQeCIOxAEFzi2k8jRozIrb3yyivJdceOHVtq20WXgk6bNi23VnS75auvvjpZX7RoUbJeNB11a2trbm3AgNrua1J/2y+++GJy3VtuuSVZ7+7urqinemDKZiA4wg4EQdiBIAg7EARhB4Ig7EAQhB0Igimb++mSSy7JrRVNPVyk6FyHJ598Mlk/ceJExdveunVrsj5r1qxkfdy4ccl6e3t7bu2mm25Krjt+/PhkvWicPnX57YwZM5LrLliwIFl/6KGHkvVmvFU1e3YgCMIOBEHYgSAIOxAEYQeCIOxAEIQdCIJx9n7q7My/U/b+/fuT67a1tSXrRbdjfuCBB5L1rq6u3FrRtfaffvppsl40Xrxv375k/bHHHsutHThwILnuXXfdlayPGTMmWU+Nww8cODC57sSJEyt+7mZ19nUMoCKEHQiCsANBEHYgCMIOBEHYgSAIOxAE4+z9dOTIkdzali1bkutOnjw5WS8asx05cmSyvmbNmtxa0dTDH3zwQbJedK18Ue9XXHFFbu3SSy9Nrls0Fl7G0aNHk/UVK1Yk62XuIdAohXt2MxtlZn82s11m9q6Z/SpbPtzMNpnZ3uz7sNq3C6BS/TmMPynpPnefKOlaSb80s4mSFkra7O6XSdqc/QygSRWG3d073X179viYpN2SRkqaJen0sc4KSbNr1COAKjij9+xmNkbSDyVtldTm7qdPGP9EUp8ngJtZu6T8G5EBqIt+fxpvZi2S1kq6x93/2rvmPXdM7POuie6+zN2nuPuUUp0CKKVfYTezQeoJ+ip3fzZb3GVmI7L6CEnNO60lgOIpm63n+ssVko64+z29lv+HpE/d/REzWyhpuLv/U8FznbVTNqcUDY2tXr06WS+a9rjoEtioiqaj7ujoyK3dfPPNyXW3b99eSUtNIW/K5v68Z79O0u2S3jGzHdmy+yU9ImmNmc2T9KGkn1WhTwA1Uhh2d39ZUt6u5YbqtgOgVjhdFgiCsANBEHYgCMIOBEHYgSAKx9mrurFzdJy9yOjRo5P15cuXJ+vTpk1L1ocOHZpba+Yx+qLbVB86dChZf/zxx5P1VatW5daKbmN9NssbZ2fPDgRB2IEgCDsQBGEHgiDsQBCEHQiCsANBMM7eBIYMGZKsT5gwIVlPXZs9derUUs89ePDgZD11i20pPdV1ahxckjZv3pysn8tj5WUwzg4ER9iBIAg7EARhB4Ig7EAQhB0IgrADQTDOfg5ITZucutZdkgYNGpSst7a2JutFfz+pqZGPHTuWXBeVYZwdCI6wA0EQdiAIwg4EQdiBIAg7EARhB4Loz/zsoyStlNQmySUtc/fHzWyJpJ9LOn1z7/vd/Y8Fz8U4O1BjeePs/Qn7CEkj3H27mbVKekPSbPXMx37c3f+zv00QdqD28sLen/nZOyV1Zo+PmdluSSOr2x6AWjuj9+xmNkbSDyVtzRYtMLO3zWy5mQ3LWafdzLaZ2bZyrQIoo9/nxptZi6QXJf2buz9rZm2SDqvnffy/qudQ/x8LnoPDeKDGKn7PLklmNkjSekl/cvdH+6iPkbTe3a8seB7CDtRYxRfCWM80oL+VtLt30LMP7k77qaSdZZsEUDv9+TR+uqSXJL0j6VS2+H5JcyVNUs9hfIekX2Qf5qWeiz07UGOlDuOrhbADtcf17EBwhB0IgrADQRB2IAjCDgRB2IEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSAKbzhZZYclfdjr54uzZc2oWXtr1r4keqtUNXv727xCXa9n/87Gzba5+5SGNZDQrL01a18SvVWqXr1xGA8EQdiBIBod9mUN3n5Ks/bWrH1J9FapuvTW0PfsAOqn0Xt2AHVC2IEgGhJ2M7vRzP5iZu+b2cJG9JDHzDrM7B0z29Ho+emyOfS6zWxnr2XDzWyTme3Nvvc5x16DeltiZgez126Hmc1sUG+jzOzPZrbLzN41s19lyxv62iX6qsvrVvf37GY2UNIeST+WdEDS65LmuvuuujaSw8w6JE1x94afgGFmfyfpuKSVp6fWMrN/l3TE3R/J/qEc5u7/3CS9LdEZTuNdo97yphm/Uw187ao5/XklGrFnnyrpfXff5+4nJK2WNKsBfTQ9d98i6ci3Fs+StCJ7vEI9fyx1l9NbU3D3Tnffnj0+Jun0NOMNfe0SfdVFI8I+UtL+Xj8fUHPN9+6SNprZG2bW3uhm+tDWa5qtTyS1NbKZPhRO411P35pmvGleu0qmPy+LD+i+a7q7T5b0D5J+mR2uNiXveQ/WTGOnSyV9Xz1zAHZK+nUjm8mmGV8r6R53/2vvWiNfuz76qsvr1oiwH5Q0qtfP38uWNQV3P5h975b0nHredjSTrtMz6Gbfuxvcz/9z9y53/9rdT0n6jRr42mXTjK+VtMrdn80WN/y166uver1ujQj765IuM7OxZjZY0hxJ6xrQx3eY2dDsgxOZ2VBJP1HzTUW9TtId2eM7JP2hgb18Q7NM4503zbga/No1fPpzd6/7l6SZ6vlE/n8l/Usjesjpa5ykt7Kvdxvdm6Sn1XNY95V6PtuYJ+kiSZsl7ZX0P5KGN1Fv/62eqb3fVk+wRjSot+nqOUR/W9KO7Gtmo1+7RF91ed04XRYIgg/ogCAIOxAEYQeCIOxAEIQdCIKwA0EQdiCI/wOiaQ3xJhNidgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from PIL import Image\n",
    "img = Image.open(\"own.png\").convert('L')\n",
    "print(img)\n",
    "import matplotlib.pyplot as plt\n",
    "plt.imshow(img, cmap='gray')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "26b9454a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[  2.2188, -12.5725,   4.7266,   2.0356,  -9.6545,   3.8602,   2.2201,\n",
       "         -12.8063,   9.1954, -10.4620]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "_img = torchvision.transforms.ToTensor()(img).reshape(1, 1, 28, 28)\n",
    "trainer.model(_img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "01d4a39f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(8)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.argmax(trainer.model(_img))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09be326c",
   "metadata": {},
   "source": [
    "### (possible) steps to improve model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03b163b9",
   "metadata": {},
   "source": [
    "1) increase ``hidden_size``  (i.e. to 64)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf635b7f",
   "metadata": {},
   "source": [
    "2) add more layer(s) and/or dropout to model:"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a581645",
   "metadata": {},
   "source": [
    "```\n",
    "        self.model = torch.nn.Sequential(\n",
    "            torch.nn.Flatten(),\n",
    "            torch.nn.Linear(channels * width * height, hidden_size),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Dropout(0.1),\n",
    "            torch.nn.Linear(hidden_size, hidden_size),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Dropout(0.1),\n",
    "            torch.nn.Linear(hidden_size, self.num_classes),\n",
    "        )\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a00d4cfd",
   "metadata": {},
   "source": [
    "3) use CNNs (spoiler for workshop part 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be8ea43c",
   "metadata": {},
   "source": [
    "```\n",
    "        self.model = torch.nn.Sequential(\n",
    "            torch.nn.Conv2d(in_channels=1, out_channels=32, kernel_size=5, stride=1, padding=2),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.MaxPool2d(2),\n",
    "            torch.nn.Conv2d(32, 64, 5, 1, 2),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.MaxPool2d(2),\n",
    "            torch.nn.Flatten(),\n",
    "            torch.nn.Linear(64 * 7 * 7, self.num_classes),\n",
    "        )\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "45947046",
   "metadata": {},
   "source": [
    "4) add color normalisation when loading MNIST (may or may not help - is in a lot of MNIST tutorials)\n",
    "```\n",
    "        transform=torchvision.transforms.Compose([\n",
    "            torchvision.transforms.ToTensor(),\n",
    "            torchvision.transforms.Normalize(\n",
    "                (0.1307,), (0.3081,))\n",
    "        ])\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac7d1090",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}