notebooks/pytorch_basics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc-hr-collapsed": true,
    "toc-nb-collapsed": true
   },
   "source": [
    "# PyTorch Basics\n",
    "- tensors like numpy\n",
    "- tensors on the gpu\n",
    "- tensors and automatic derivatives\n",
    "- tensors as neural network abstractions: `torch.nn`\n",
    "- optimizers: `nn.optim`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Init, helpers, utils, ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:57.872710Z",
     "iopub.status.busy": "2020-11-27T14:29:57.872327Z",
     "iopub.status.idle": "2020-11-27T14:29:58.248781Z",
     "shell.execute_reply": "2020-11-27T14:29:58.248447Z"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torchvision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.251518Z",
     "iopub.status.busy": "2020-11-27T14:29:58.251163Z",
     "iopub.status.idle": "2020-11-27T14:29:58.371152Z",
     "shell.execute_reply": "2020-11-27T14:29:58.370772Z"
    }
   },
   "outputs": [],
   "source": [
    "from pprint import pprint\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from IPython.core.debugger import set_trace"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "toc-hr-collapsed": true,
    "toc-nb-collapsed": true
   },
   "source": [
    "# Tensors\n",
    "tensors - the atoms of machine learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tensors in numpy and pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.373886Z",
     "iopub.status.busy": "2020-11-27T14:29:58.373437Z",
     "iopub.status.idle": "2020-11-27T14:29:58.376006Z",
     "shell.execute_reply": "2020-11-27T14:29:58.375601Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from numpy.linalg import inv\n",
    "from numpy.linalg import multi_dot as mdot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.379403Z",
     "iopub.status.busy": "2020-11-27T14:29:58.378853Z",
     "iopub.status.idle": "2020-11-27T14:29:58.381591Z",
     "shell.execute_reply": "2020-11-27T14:29:58.381209Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 0., 0.],\n",
       "       [0., 1., 0.],\n",
       "       [0., 0., 1.]])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# numpy\n",
    "np.eye(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.383910Z",
     "iopub.status.busy": "2020-11-27T14:29:58.383497Z",
     "iopub.status.idle": "2020-11-27T14:29:58.387241Z",
     "shell.execute_reply": "2020-11-27T14:29:58.386867Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 0., 0.],\n",
       "        [0., 1., 0.],\n",
       "        [0., 0., 1.]])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# torch\n",
    "torch.eye(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.390155Z",
     "iopub.status.busy": "2020-11-27T14:29:58.389651Z",
     "iopub.status.idle": "2020-11-27T14:29:58.392675Z",
     "shell.execute_reply": "2020-11-27T14:29:58.392197Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.01761407, 0.22226662, 0.10141616],\n",
       "       [0.29994794, 0.52538242, 0.63938118],\n",
       "       [0.67686391, 0.6255289 , 0.12404051],\n",
       "       [0.97030623, 0.90453374, 0.0239299 ],\n",
       "       [0.19845467, 0.00667336, 0.77244887]])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# numpy\n",
    "X = np.random.random((5, 3))\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.395477Z",
     "iopub.status.busy": "2020-11-27T14:29:58.395082Z",
     "iopub.status.idle": "2020-11-27T14:29:58.397919Z",
     "shell.execute_reply": "2020-11-27T14:29:58.398229Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.2812, 0.8975, 0.3753],\n",
       "        [0.1442, 0.3561, 0.7479],\n",
       "        [0.8167, 0.1642, 0.0646],\n",
       "        [0.8707, 0.1257, 0.0439],\n",
       "        [0.5695, 0.9224, 0.2349]])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# pytorch\n",
    "Y = torch.rand((5, 3))\n",
    "Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.400827Z",
     "iopub.status.busy": "2020-11-27T14:29:58.400434Z",
     "iopub.status.idle": "2020-11-27T14:29:58.402908Z",
     "shell.execute_reply": "2020-11-27T14:29:58.402566Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 3)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.405404Z",
     "iopub.status.busy": "2020-11-27T14:29:58.405021Z",
     "iopub.status.idle": "2020-11-27T14:29:58.407423Z",
     "shell.execute_reply": "2020-11-27T14:29:58.407029Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([5, 3])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.410130Z",
     "iopub.status.busy": "2020-11-27T14:29:58.409736Z",
     "iopub.status.idle": "2020-11-27T14:29:58.411943Z",
     "shell.execute_reply": "2020-11-27T14:29:58.411563Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.52930221, 1.46389941, 0.45404138],\n",
       "       [1.46389941, 1.53494135, 0.46285221],\n",
       "       [0.45404138, 0.46285221, 1.03172948]])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# numpy\n",
    "X.T @ X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.414461Z",
     "iopub.status.busy": "2020-11-27T14:29:58.413978Z",
     "iopub.status.idle": "2020-11-27T14:29:58.416714Z",
     "shell.execute_reply": "2020-11-27T14:29:58.417016Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1.8492, 1.0725, 0.4381],\n",
       "        [1.0725, 1.8258, 0.8359],\n",
       "        [0.4381, 0.8359, 0.7614]])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# torch\n",
    "Y.t() @ Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.420094Z",
     "iopub.status.busy": "2020-11-27T14:29:58.419576Z",
     "iopub.status.idle": "2020-11-27T14:29:58.421941Z",
     "shell.execute_reply": "2020-11-27T14:29:58.421621Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 7.52001688, -7.139912  , -0.1063019 ],\n",
       "       [-7.139912  ,  7.53243033, -0.23706458],\n",
       "       [-0.1063019 , -0.23706458,  1.12237883]])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# numpy\n",
    "inv(X.T @ X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.424461Z",
     "iopub.status.busy": "2020-11-27T14:29:58.423982Z",
     "iopub.status.idle": "2020-11-27T14:29:58.427118Z",
     "shell.execute_reply": "2020-11-27T14:29:58.426708Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.8252, -0.5376,  0.1153],\n",
       "        [-0.5376,  1.4514, -1.2841],\n",
       "        [ 0.1153, -1.2841,  2.6567]])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# torch\n",
    "torch.inverse(Y.t() @ Y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## More on PyTorch Tensors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Operations are also available as methods."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.429906Z",
     "iopub.status.busy": "2020-11-27T14:29:58.429394Z",
     "iopub.status.idle": "2020-11-27T14:29:58.432350Z",
     "shell.execute_reply": "2020-11-27T14:29:58.432025Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[2., 1., 1.],\n",
       "        [1., 2., 1.],\n",
       "        [1., 1., 2.]])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A = torch.eye(3)\n",
    "A.add(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.434883Z",
     "iopub.status.busy": "2020-11-27T14:29:58.434393Z",
     "iopub.status.idle": "2020-11-27T14:29:58.437422Z",
     "shell.execute_reply": "2020-11-27T14:29:58.437036Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 0., 0.],\n",
       "        [0., 1., 0.],\n",
       "        [0., 0., 1.]])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Any operation that mutates a tensor in-place has a `_` suffix."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.440000Z",
     "iopub.status.busy": "2020-11-27T14:29:58.439512Z",
     "iopub.status.idle": "2020-11-27T14:29:58.442612Z",
     "shell.execute_reply": "2020-11-27T14:29:58.442199Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[2., 1., 1.],\n",
       "        [1., 2., 1.],\n",
       "        [1., 1., 2.]])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A.add_(1)\n",
    "A"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Indexing and broadcasting\n",
    "It works as expected/like numpy:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.445318Z",
     "iopub.status.busy": "2020-11-27T14:29:58.444806Z",
     "iopub.status.idle": "2020-11-27T14:29:58.447381Z",
     "shell.execute_reply": "2020-11-27T14:29:58.447684Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(2.)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A[0, 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.450172Z",
     "iopub.status.busy": "2020-11-27T14:29:58.449690Z",
     "iopub.status.idle": "2020-11-27T14:29:58.453344Z",
     "shell.execute_reply": "2020-11-27T14:29:58.452765Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([2., 1., 1.])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.456045Z",
     "iopub.status.busy": "2020-11-27T14:29:58.455552Z",
     "iopub.status.idle": "2020-11-27T14:29:58.458491Z",
     "shell.execute_reply": "2020-11-27T14:29:58.458149Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[2., 1., 1.],\n",
       "        [1., 2., 1.]])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A[0:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.460819Z",
     "iopub.status.busy": "2020-11-27T14:29:58.460376Z",
     "iopub.status.idle": "2020-11-27T14:29:58.463261Z",
     "shell.execute_reply": "2020-11-27T14:29:58.463589Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 1.],\n",
       "        [2., 1.],\n",
       "        [1., 2.]])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A[:, 1:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Converting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.466125Z",
     "iopub.status.busy": "2020-11-27T14:29:58.465655Z",
     "iopub.status.idle": "2020-11-27T14:29:58.468190Z",
     "shell.execute_reply": "2020-11-27T14:29:58.468465Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 0., 0.],\n",
       "        [0., 1., 0.],\n",
       "        [0., 0., 1.]])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A = torch.eye(3)\n",
    "A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.471229Z",
     "iopub.status.busy": "2020-11-27T14:29:58.470722Z",
     "iopub.status.idle": "2020-11-27T14:29:58.472972Z",
     "shell.execute_reply": "2020-11-27T14:29:58.473299Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 0., 0.],\n",
       "       [0., 1., 0.],\n",
       "       [0., 0., 1.]], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# torch --> numpy\n",
    "B = A.numpy()\n",
    "B"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: torch and numpy can share the same memory / zero-copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.475954Z",
     "iopub.status.busy": "2020-11-27T14:29:58.475482Z",
     "iopub.status.idle": "2020-11-27T14:29:58.477945Z",
     "shell.execute_reply": "2020-11-27T14:29:58.478307Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1.5000, 0.5000, 0.5000],\n",
       "        [0.5000, 1.5000, 0.5000],\n",
       "        [0.5000, 0.5000, 1.5000]])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A.add_(.5)\n",
    "A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.481017Z",
     "iopub.status.busy": "2020-11-27T14:29:58.480521Z",
     "iopub.status.idle": "2020-11-27T14:29:58.483433Z",
     "shell.execute_reply": "2020-11-27T14:29:58.483090Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.5, 0.5, 0.5],\n",
       "       [0.5, 1.5, 0.5],\n",
       "       [0.5, 0.5, 1.5]], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "B"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.486141Z",
     "iopub.status.busy": "2020-11-27T14:29:58.485632Z",
     "iopub.status.idle": "2020-11-27T14:29:58.488665Z",
     "shell.execute_reply": "2020-11-27T14:29:58.488329Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 0., 0.],\n",
       "        [0., 1., 0.],\n",
       "        [0., 0., 1.]], dtype=torch.float64)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# numpy --> torch\n",
    "torch.from_numpy(np.eye(3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Much more"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.497961Z",
     "iopub.status.busy": "2020-11-27T14:29:58.497557Z",
     "iopub.status.idle": "2020-11-27T14:29:58.501011Z",
     "shell.execute_reply": "2020-11-27T14:29:58.500603Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['AVG',\n",
       " 'AggregationType',\n",
       " 'AnyType',\n",
       " 'Argument',\n",
       " 'ArgumentSpec',\n",
       " 'Assert',\n",
       " 'BFloat16Storage',\n",
       " 'BFloat16Tensor',\n",
       " 'BenchmarkConfig',\n",
       " 'BenchmarkExecutionStats',\n",
       " 'Block',\n",
       " 'BoolStorage',\n",
       " 'BoolTensor',\n",
       " 'BoolType',\n",
       " 'BufferDict',\n",
       " 'ByteStorage',\n",
       " 'ByteTensor',\n",
       " 'CONV_BN_FUSION',\n",
       " 'CallStack',\n",
       " 'Capsule',\n",
       " 'CharStorage',\n",
       " 'CharTensor',\n",
       " 'ClassType',\n",
       " 'Code',\n",
       " 'CompilationUnit',\n",
       " 'CompleteArgumentSpec',\n",
       " 'ComplexDoubleStorage',\n",
       " 'ComplexFloatStorage',\n",
       " 'ConcreteModuleType',\n",
       " 'ConcreteModuleTypeBuilder',\n",
       " 'CudaBFloat16StorageBase',\n",
       " 'CudaBoolStorageBase',\n",
       " 'CudaByteStorageBase',\n",
       " 'CudaCharStorageBase',\n",
       " 'CudaComplexDoubleStorageBase',\n",
       " 'CudaComplexFloatStorageBase',\n",
       " 'CudaDoubleStorageBase',\n",
       " 'CudaFloatStorageBase',\n",
       " 'CudaHalfStorageBase',\n",
       " 'CudaIntStorageBase',\n",
       " 'CudaLongStorageBase',\n",
       " 'CudaShortStorageBase',\n",
       " 'DeepCopyMemoTable',\n",
       " 'DeviceObjType',\n",
       " 'DictType',\n",
       " 'DisableTorchFunction',\n",
       " 'DoubleStorage',\n",
       " 'DoubleTensor',\n",
       " 'EnumType',\n",
       " 'ErrorReport',\n",
       " 'ExecutionPlan',\n",
       " 'FUSE_ADD_RELU',\n",
       " 'FatalError',\n",
       " 'FileCheck',\n",
       " 'FloatStorage',\n",
       " 'FloatTensor',\n",
       " 'FloatType',\n",
       " 'FunctionSchema',\n",
       " 'Future',\n",
       " 'FutureType',\n",
       " 'Generator',\n",
       " 'Gradient',\n",
       " 'Graph',\n",
       " 'GraphExecutorState',\n",
       " 'HOIST_CONV_PACKED_PARAMS',\n",
       " 'HalfStorage',\n",
       " 'HalfStorageBase',\n",
       " 'HalfTensor',\n",
       " 'INSERT_FOLD_PREPACK_OPS',\n",
       " 'IODescriptor',\n",
       " 'IntStorage',\n",
       " 'IntTensor',\n",
       " 'IntType',\n",
       " 'InterfaceType',\n",
       " 'JITException',\n",
       " 'ListType',\n",
       " 'LiteScriptModule',\n",
       " 'LockingLogger',\n",
       " 'LoggerBase',\n",
       " 'LongStorage',\n",
       " 'LongTensor',\n",
       " 'MobileOptimizerType',\n",
       " 'ModuleDict',\n",
       " 'Node',\n",
       " 'NoneType',\n",
       " 'NoopLogger',\n",
       " 'NumberType',\n",
       " 'OptionalType',\n",
       " 'ParameterDict',\n",
       " 'PyObjectType',\n",
       " 'PyTorchFileReader',\n",
       " 'PyTorchFileWriter',\n",
       " 'QInt32Storage',\n",
       " 'QInt32StorageBase',\n",
       " 'QInt8Storage',\n",
       " 'QInt8StorageBase',\n",
       " 'QUInt8Storage',\n",
       " 'REMOVE_DROPOUT',\n",
       " 'RRefType',\n",
       " 'SUM',\n",
       " 'ScriptClass',\n",
       " 'ScriptFunction',\n",
       " 'ScriptMethod',\n",
       " 'ScriptModule',\n",
       " 'ScriptObject',\n",
       " 'Set',\n",
       " 'ShortStorage',\n",
       " 'ShortTensor',\n",
       " 'Size',\n",
       " 'StaticRuntime',\n",
       " 'Storage',\n",
       " 'StringType',\n",
       " 'TYPE_CHECKING',\n",
       " 'Tensor',\n",
       " 'TensorType',\n",
       " 'ThroughputBenchmark',\n",
       " 'TracingState',\n",
       " 'TupleType',\n",
       " 'Type',\n",
       " 'USE_GLOBAL_DEPS',\n",
       " 'USE_RTLD_GLOBAL_WITH_LIBTORCH',\n",
       " 'Use',\n",
       " 'Value',\n",
       " 'abs',\n",
       " 'abs_',\n",
       " 'absolute',\n",
       " 'acos',\n",
       " 'acos_',\n",
       " 'acosh',\n",
       " 'acosh_',\n",
       " 'adaptive_avg_pool1d',\n",
       " 'adaptive_max_pool1d',\n",
       " 'add',\n",
       " 'addbmm',\n",
       " 'addcdiv',\n",
       " 'addcmul',\n",
       " 'addmm',\n",
       " 'addmv',\n",
       " 'addmv_',\n",
       " 'addr',\n",
       " 'affine_grid_generator',\n",
       " 'align_tensors',\n",
       " 'all',\n",
       " 'allclose',\n",
       " 'alpha_dropout',\n",
       " 'alpha_dropout_',\n",
       " 'amax',\n",
       " 'amin',\n",
       " 'angle',\n",
       " 'any',\n",
       " 'arange',\n",
       " 'arccos',\n",
       " 'arccos_',\n",
       " 'arccosh',\n",
       " 'arccosh_',\n",
       " 'arcsin',\n",
       " 'arcsin_',\n",
       " 'arcsinh',\n",
       " 'arcsinh_',\n",
       " 'arctan',\n",
       " 'arctan_',\n",
       " 'arctanh',\n",
       " 'arctanh_',\n",
       " 'argmax',\n",
       " 'argmin',\n",
       " 'argsort',\n",
       " 'as_strided',\n",
       " 'as_strided_',\n",
       " 'as_tensor',\n",
       " 'asin',\n",
       " 'asin_',\n",
       " 'asinh',\n",
       " 'asinh_',\n",
       " 'atan',\n",
       " 'atan2',\n",
       " 'atan_',\n",
       " 'atanh',\n",
       " 'atanh_',\n",
       " 'atleast_1d',\n",
       " 'atleast_2d',\n",
       " 'atleast_3d',\n",
       " 'autocast_decrement_nesting',\n",
       " 'autocast_increment_nesting',\n",
       " 'autograd',\n",
       " 'avg_pool1d',\n",
       " 'backends',\n",
       " 'baddbmm',\n",
       " 'bartlett_window',\n",
       " 'batch_norm',\n",
       " 'batch_norm_backward_elemt',\n",
       " 'batch_norm_backward_reduce',\n",
       " 'batch_norm_elemt',\n",
       " 'batch_norm_gather_stats',\n",
       " 'batch_norm_gather_stats_with_counts',\n",
       " 'batch_norm_stats',\n",
       " 'batch_norm_update_stats',\n",
       " 'bernoulli',\n",
       " 'bfloat16',\n",
       " 'bilinear',\n",
       " 'binary_cross_entropy_with_logits',\n",
       " 'bincount',\n",
       " 'binomial',\n",
       " 'bitwise_and',\n",
       " 'bitwise_not',\n",
       " 'bitwise_or',\n",
       " 'bitwise_xor',\n",
       " 'blackman_window',\n",
       " 'block_diag',\n",
       " 'bmm',\n",
       " 'bool',\n",
       " 'broadcast_tensors',\n",
       " 'bucketize',\n",
       " 'can_cast',\n",
       " 'cartesian_prod',\n",
       " 'cat',\n",
       " 'cdist',\n",
       " 'cdouble',\n",
       " 'ceil',\n",
       " 'ceil_',\n",
       " 'celu',\n",
       " 'celu_',\n",
       " 'cfloat',\n",
       " 'chain_matmul',\n",
       " 'channel_shuffle',\n",
       " 'channels_last',\n",
       " 'channels_last_3d',\n",
       " 'cholesky',\n",
       " 'cholesky_inverse',\n",
       " 'cholesky_solve',\n",
       " 'choose_qparams_optimized',\n",
       " 'chunk',\n",
       " 'clamp',\n",
       " 'clamp_',\n",
       " 'clamp_max',\n",
       " 'clamp_max_',\n",
       " 'clamp_min',\n",
       " 'clamp_min_',\n",
       " 'classes',\n",
       " 'clear_autocast_cache',\n",
       " 'clip',\n",
       " 'clip_',\n",
       " 'clone',\n",
       " 'combinations',\n",
       " 'compiled_with_cxx11_abi',\n",
       " 'complex',\n",
       " 'complex128',\n",
       " 'complex32',\n",
       " 'complex64',\n",
       " 'conj',\n",
       " 'constant_pad_nd',\n",
       " 'contiguous_format',\n",
       " 'conv1d',\n",
       " 'conv2d',\n",
       " 'conv3d',\n",
       " 'conv_tbc',\n",
       " 'conv_transpose1d',\n",
       " 'conv_transpose2d',\n",
       " 'conv_transpose3d',\n",
       " 'convolution',\n",
       " 'cos',\n",
       " 'cos_',\n",
       " 'cosh',\n",
       " 'cosh_',\n",
       " 'cosine_embedding_loss',\n",
       " 'cosine_similarity',\n",
       " 'count_nonzero',\n",
       " 'cpp',\n",
       " 'cross',\n",
       " 'ctc_loss',\n",
       " 'ctypes',\n",
       " 'cuda',\n",
       " 'cudnn_affine_grid_generator',\n",
       " 'cudnn_batch_norm',\n",
       " 'cudnn_convolution',\n",
       " 'cudnn_convolution_transpose',\n",
       " 'cudnn_grid_sampler',\n",
       " 'cudnn_is_acceptable',\n",
       " 'cummax',\n",
       " 'cummin',\n",
       " 'cumprod',\n",
       " 'cumsum',\n",
       " 'default_generator',\n",
       " 'deg2rad',\n",
       " 'deg2rad_',\n",
       " 'dequantize',\n",
       " 'det',\n",
       " 'detach',\n",
       " 'detach_',\n",
       " 'device',\n",
       " 'diag',\n",
       " 'diag_embed',\n",
       " 'diagflat',\n",
       " 'diagonal',\n",
       " 'digamma',\n",
       " 'dist',\n",
       " 'distributed',\n",
       " 'distributions',\n",
       " 'div',\n",
       " 'divide',\n",
       " 'dot',\n",
       " 'double',\n",
       " 'dropout',\n",
       " 'dropout_',\n",
       " 'dsmm',\n",
       " 'dstack',\n",
       " 'dtype',\n",
       " 'eig',\n",
       " 'einsum',\n",
       " 'embedding',\n",
       " 'embedding_bag',\n",
       " 'embedding_renorm_',\n",
       " 'empty',\n",
       " 'empty_like',\n",
       " 'empty_meta',\n",
       " 'empty_quantized',\n",
       " 'empty_strided',\n",
       " 'enable_grad',\n",
       " 'eq',\n",
       " 'equal',\n",
       " 'erf',\n",
       " 'erf_',\n",
       " 'erfc',\n",
       " 'erfc_',\n",
       " 'erfinv',\n",
       " 'exp',\n",
       " 'exp2',\n",
       " 'exp2_',\n",
       " 'exp_',\n",
       " 'expm1',\n",
       " 'expm1_',\n",
       " 'eye',\n",
       " 'fake_quantize_per_channel_affine',\n",
       " 'fake_quantize_per_tensor_affine',\n",
       " 'fbgemm_linear_fp16_weight',\n",
       " 'fbgemm_linear_fp16_weight_fp32_activation',\n",
       " 'fbgemm_linear_int8_weight',\n",
       " 'fbgemm_linear_int8_weight_fp32_activation',\n",
       " 'fbgemm_linear_quantize_weight',\n",
       " 'fbgemm_pack_gemm_matrix_fp16',\n",
       " 'fbgemm_pack_quantized_matrix',\n",
       " 'feature_alpha_dropout',\n",
       " 'feature_alpha_dropout_',\n",
       " 'feature_dropout',\n",
       " 'feature_dropout_',\n",
       " 'fft',\n",
       " 'fill_',\n",
       " 'finfo',\n",
       " 'fix',\n",
       " 'fix_',\n",
       " 'flatten',\n",
       " 'flip',\n",
       " 'fliplr',\n",
       " 'flipud',\n",
       " 'float',\n",
       " 'float16',\n",
       " 'float32',\n",
       " 'float64',\n",
       " 'floor',\n",
       " 'floor_',\n",
       " 'floor_divide',\n",
       " 'fmod',\n",
       " 'fork',\n",
       " 'frac',\n",
       " 'frac_',\n",
       " 'frobenius_norm',\n",
       " 'from_file',\n",
       " 'from_numpy',\n",
       " 'full',\n",
       " 'full_like',\n",
       " 'functional',\n",
       " 'futures',\n",
       " 'gather',\n",
       " 'gcd',\n",
       " 'gcd_',\n",
       " 'ge',\n",
       " 'geqrf',\n",
       " 'ger',\n",
       " 'get_default_dtype',\n",
       " 'get_device',\n",
       " 'get_file_path',\n",
       " 'get_num_interop_threads',\n",
       " 'get_num_threads',\n",
       " 'get_rng_state',\n",
       " 'greater',\n",
       " 'greater_equal',\n",
       " 'grid_sampler',\n",
       " 'grid_sampler_2d',\n",
       " 'grid_sampler_3d',\n",
       " 'group_norm',\n",
       " 'gru',\n",
       " 'gru_cell',\n",
       " 'gt',\n",
       " 'half',\n",
       " 'hamming_window',\n",
       " 'hann_window',\n",
       " 'hardshrink',\n",
       " 'has_cuda',\n",
       " 'has_cudnn',\n",
       " 'has_lapack',\n",
       " 'has_mkl',\n",
       " 'has_mkldnn',\n",
       " 'has_openmp',\n",
       " 'heaviside',\n",
       " 'hinge_embedding_loss',\n",
       " 'histc',\n",
       " 'hsmm',\n",
       " 'hspmm',\n",
       " 'hstack',\n",
       " 'hub',\n",
       " 'hypot',\n",
       " 'i0',\n",
       " 'i0_',\n",
       " 'ifft',\n",
       " 'iinfo',\n",
       " 'imag',\n",
       " 'import_ir_module',\n",
       " 'import_ir_module_from_buffer',\n",
       " 'index_add',\n",
       " 'index_copy',\n",
       " 'index_fill',\n",
       " 'index_put',\n",
       " 'index_put_',\n",
       " 'index_select',\n",
       " 'init_num_threads',\n",
       " 'initial_seed',\n",
       " 'instance_norm',\n",
       " 'int',\n",
       " 'int16',\n",
       " 'int32',\n",
       " 'int64',\n",
       " 'int8',\n",
       " 'int_repr',\n",
       " 'inverse',\n",
       " 'irfft',\n",
       " 'is_anomaly_enabled',\n",
       " 'is_autocast_enabled',\n",
       " 'is_complex',\n",
       " 'is_deterministic',\n",
       " 'is_distributed',\n",
       " 'is_floating_point',\n",
       " 'is_grad_enabled',\n",
       " 'is_nonzero',\n",
       " 'is_same_size',\n",
       " 'is_signed',\n",
       " 'is_storage',\n",
       " 'is_tensor',\n",
       " 'is_vulkan_available',\n",
       " 'isclose',\n",
       " 'isfinite',\n",
       " 'isinf',\n",
       " 'isnan',\n",
       " 'isneginf',\n",
       " 'isposinf',\n",
       " 'isreal',\n",
       " 'istft',\n",
       " 'jit',\n",
       " 'kaiser_window',\n",
       " 'kl_div',\n",
       " 'kthvalue',\n",
       " 'layer_norm',\n",
       " 'layout',\n",
       " 'lcm',\n",
       " 'lcm_',\n",
       " 'le',\n",
       " 'legacy_contiguous_format',\n",
       " 'lerp',\n",
       " 'less',\n",
       " 'less_equal',\n",
       " 'lgamma',\n",
       " 'linalg',\n",
       " 'linspace',\n",
       " 'load',\n",
       " 'lobpcg',\n",
       " 'log',\n",
       " 'log10',\n",
       " 'log10_',\n",
       " 'log1p',\n",
       " 'log1p_',\n",
       " 'log2',\n",
       " 'log2_',\n",
       " 'log_',\n",
       " 'log_softmax',\n",
       " 'logaddexp',\n",
       " 'logaddexp2',\n",
       " 'logcumsumexp',\n",
       " 'logdet',\n",
       " 'logical_and',\n",
       " 'logical_not',\n",
       " 'logical_or',\n",
       " 'logical_xor',\n",
       " 'logit',\n",
       " 'logit_',\n",
       " 'logspace',\n",
       " 'logsumexp',\n",
       " 'long',\n",
       " 'lstm',\n",
       " 'lstm_cell',\n",
       " 'lstsq',\n",
       " 'lt',\n",
       " 'lu',\n",
       " 'lu_solve',\n",
       " 'lu_unpack',\n",
       " 'manual_seed',\n",
       " 'margin_ranking_loss',\n",
       " 'masked_fill',\n",
       " 'masked_scatter',\n",
       " 'masked_select',\n",
       " 'matmul',\n",
       " 'matrix_exp',\n",
       " 'matrix_power',\n",
       " 'matrix_rank',\n",
       " 'max',\n",
       " 'max_pool1d',\n",
       " 'max_pool1d_with_indices',\n",
       " 'max_pool2d',\n",
       " 'max_pool3d',\n",
       " 'maximum',\n",
       " 'mean',\n",
       " 'median',\n",
       " 'memory_format',\n",
       " 'merge_type_from_type_comment',\n",
       " 'meshgrid',\n",
       " 'min',\n",
       " 'minimum',\n",
       " 'miopen_batch_norm',\n",
       " 'miopen_convolution',\n",
       " 'miopen_convolution_transpose',\n",
       " 'miopen_depthwise_convolution',\n",
       " 'miopen_rnn',\n",
       " 'mkldnn_adaptive_avg_pool2d',\n",
       " 'mkldnn_convolution',\n",
       " 'mkldnn_convolution_backward_weights',\n",
       " 'mkldnn_max_pool2d',\n",
       " 'mkldnn_max_pool3d',\n",
       " 'mm',\n",
       " 'mode',\n",
       " 'movedim',\n",
       " 'mul',\n",
       " 'multinomial',\n",
       " 'multiply',\n",
       " 'multiprocessing',\n",
       " 'mv',\n",
       " 'mvlgamma',\n",
       " 'name',\n",
       " 'nanquantile',\n",
       " 'nansum',\n",
       " 'narrow',\n",
       " 'native_batch_norm',\n",
       " 'native_group_norm',\n",
       " 'native_layer_norm',\n",
       " 'native_norm',\n",
       " 'ne',\n",
       " 'neg',\n",
       " 'neg_',\n",
       " 'negative',\n",
       " 'negative_',\n",
       " 'nextafter',\n",
       " 'nn',\n",
       " 'no_grad',\n",
       " 'nonzero',\n",
       " 'norm',\n",
       " 'norm_except_dim',\n",
       " 'normal',\n",
       " 'not_equal',\n",
       " 'nuclear_norm',\n",
       " 'numel',\n",
       " 'ones',\n",
       " 'ones_like',\n",
       " 'onnx',\n",
       " 'ops',\n",
       " 'optim',\n",
       " 'orgqr',\n",
       " 'ormqr',\n",
       " 'os',\n",
       " 'outer',\n",
       " 'overrides',\n",
       " 'pairwise_distance',\n",
       " 'parse_ir',\n",
       " 'parse_schema',\n",
       " 'parse_type_comment',\n",
       " 'pca_lowrank',\n",
       " 'pdist',\n",
       " 'per_channel_affine',\n",
       " 'per_channel_affine_float_qparams',\n",
       " 'per_channel_symmetric',\n",
       " 'per_tensor_affine',\n",
       " 'per_tensor_symmetric',\n",
       " 'pinverse',\n",
       " 'pixel_shuffle',\n",
       " 'platform',\n",
       " 'poisson',\n",
       " 'poisson_nll_loss',\n",
       " 'polar',\n",
       " 'polygamma',\n",
       " 'pow',\n",
       " 'prelu',\n",
       " 'prepare_multiprocessing_environment',\n",
       " 'preserve_format',\n",
       " 'prod',\n",
       " 'promote_types',\n",
       " 'q_per_channel_axis',\n",
       " 'q_per_channel_scales',\n",
       " 'q_per_channel_zero_points',\n",
       " 'q_scale',\n",
       " 'q_zero_point',\n",
       " 'qint32',\n",
       " 'qint8',\n",
       " 'qr',\n",
       " 'qscheme',\n",
       " 'quantile',\n",
       " 'quantization',\n",
       " 'quantize_per_channel',\n",
       " 'quantize_per_tensor',\n",
       " 'quantized_batch_norm',\n",
       " 'quantized_gru',\n",
       " 'quantized_gru_cell',\n",
       " 'quantized_lstm',\n",
       " 'quantized_lstm_cell',\n",
       " 'quantized_max_pool1d',\n",
       " 'quantized_max_pool2d',\n",
       " 'quantized_rnn_relu_cell',\n",
       " 'quantized_rnn_tanh_cell',\n",
       " 'quasirandom',\n",
       " 'quint8',\n",
       " 'rad2deg',\n",
       " 'rad2deg_',\n",
       " 'rand',\n",
       " 'rand_like',\n",
       " 'randint',\n",
       " 'randint_like',\n",
       " 'randn',\n",
       " 'randn_like',\n",
       " 'random',\n",
       " 'randperm',\n",
       " 'range',\n",
       " 'real',\n",
       " 'reciprocal',\n",
       " 'reciprocal_',\n",
       " 'relu',\n",
       " 'relu_',\n",
       " 'remainder',\n",
       " 'renorm',\n",
       " 'repeat_interleave',\n",
       " 'reshape',\n",
       " 'resize_as_',\n",
       " 'result_type',\n",
       " 'rfft',\n",
       " 'rnn_relu',\n",
       " 'rnn_relu_cell',\n",
       " 'rnn_tanh',\n",
       " 'rnn_tanh_cell',\n",
       " 'roll',\n",
       " 'rot90',\n",
       " 'round',\n",
       " 'round_',\n",
       " 'rrelu',\n",
       " 'rrelu_',\n",
       " 'rsqrt',\n",
       " 'rsqrt_',\n",
       " 'rsub',\n",
       " 'saddmm',\n",
       " 'save',\n",
       " 'scalar_tensor',\n",
       " 'scatter',\n",
       " 'scatter_add',\n",
       " 'searchsorted',\n",
       " 'seed',\n",
       " 'select',\n",
       " 'selu',\n",
       " 'selu_',\n",
       " 'serialization',\n",
       " 'set_anomaly_enabled',\n",
       " 'set_autocast_enabled',\n",
       " 'set_default_dtype',\n",
       " 'set_default_tensor_type',\n",
       " 'set_deterministic',\n",
       " 'set_flush_denormal',\n",
       " 'set_grad_enabled',\n",
       " 'set_num_interop_threads',\n",
       " 'set_num_threads',\n",
       " 'set_printoptions',\n",
       " 'set_rng_state',\n",
       " 'sgn',\n",
       " 'short',\n",
       " 'sigmoid',\n",
       " 'sigmoid_',\n",
       " 'sign',\n",
       " 'signbit',\n",
       " 'sin',\n",
       " 'sin_',\n",
       " 'sinh',\n",
       " 'sinh_',\n",
       " 'slogdet',\n",
       " 'smm',\n",
       " 'softmax',\n",
       " 'solve',\n",
       " 'sort',\n",
       " 'sparse',\n",
       " 'sparse_coo',\n",
       " 'sparse_coo_tensor',\n",
       " 'split',\n",
       " 'split_with_sizes',\n",
       " 'spmm',\n",
       " 'sqrt',\n",
       " 'sqrt_',\n",
       " 'square',\n",
       " 'square_',\n",
       " 'squeeze',\n",
       " 'sspaddmm',\n",
       " 'stack',\n",
       " 'std',\n",
       " 'std_mean',\n",
       " 'stft',\n",
       " 'storage',\n",
       " 'strided',\n",
       " 'sub',\n",
       " 'subtract',\n",
       " 'sum',\n",
       " 'svd',\n",
       " 'svd_lowrank',\n",
       " 'symeig',\n",
       " 'sys',\n",
       " 't',\n",
       " 'take',\n",
       " 'tan',\n",
       " 'tan_',\n",
       " 'tanh',\n",
       " 'tanh_',\n",
       " 'tensor',\n",
       " 'tensordot',\n",
       " 'testing',\n",
       " 'textwrap',\n",
       " 'threshold',\n",
       " 'threshold_',\n",
       " 'topk',\n",
       " 'torch',\n",
       " 'trace',\n",
       " 'transpose',\n",
       " 'trapz',\n",
       " 'triangular_solve',\n",
       " 'tril',\n",
       " 'tril_indices',\n",
       " 'triplet_margin_loss',\n",
       " 'triu',\n",
       " 'triu_indices',\n",
       " 'true_divide',\n",
       " 'trunc',\n",
       " 'trunc_',\n",
       " 'typename',\n",
       " 'types',\n",
       " 'uint8',\n",
       " 'unbind',\n",
       " 'unify_type_list',\n",
       " 'unique',\n",
       " 'unique_consecutive',\n",
       " 'unsafe_chunk',\n",
       " 'unsafe_split',\n",
       " 'unsafe_split_with_sizes',\n",
       " 'unsqueeze',\n",
       " 'utils',\n",
       " 'vander',\n",
       " 'var',\n",
       " 'var_mean',\n",
       " 'vdot',\n",
       " 'version',\n",
       " 'view_as_complex',\n",
       " 'view_as_real',\n",
       " 'vstack',\n",
       " 'wait',\n",
       " 'where',\n",
       " 'zero_',\n",
       " 'zeros',\n",
       " 'zeros_like']"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[o for o in dir(torch) if not o.startswith(\"_\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.508366Z",
     "iopub.status.busy": "2020-11-27T14:29:58.507900Z",
     "iopub.status.idle": "2020-11-27T14:29:58.510636Z",
     "shell.execute_reply": "2020-11-27T14:29:58.510290Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['T',\n",
       " 'abs',\n",
       " 'abs_',\n",
       " 'absolute',\n",
       " 'absolute_',\n",
       " 'acos',\n",
       " 'acos_',\n",
       " 'acosh',\n",
       " 'acosh_',\n",
       " 'add',\n",
       " 'add_',\n",
       " 'addbmm',\n",
       " 'addbmm_',\n",
       " 'addcdiv',\n",
       " 'addcdiv_',\n",
       " 'addcmul',\n",
       " 'addcmul_',\n",
       " 'addmm',\n",
       " 'addmm_',\n",
       " 'addmv',\n",
       " 'addmv_',\n",
       " 'addr',\n",
       " 'addr_',\n",
       " 'align_as',\n",
       " 'align_to',\n",
       " 'all',\n",
       " 'allclose',\n",
       " 'amax',\n",
       " 'amin',\n",
       " 'angle',\n",
       " 'any',\n",
       " 'apply_',\n",
       " 'arccos',\n",
       " 'arccos_',\n",
       " 'arccosh',\n",
       " 'arccosh_',\n",
       " 'arcsin',\n",
       " 'arcsin_',\n",
       " 'arcsinh',\n",
       " 'arcsinh_',\n",
       " 'arctan',\n",
       " 'arctan_',\n",
       " 'arctanh',\n",
       " 'arctanh_',\n",
       " 'argmax',\n",
       " 'argmin',\n",
       " 'argsort',\n",
       " 'as_strided',\n",
       " 'as_strided_',\n",
       " 'as_subclass',\n",
       " 'asin',\n",
       " 'asin_',\n",
       " 'asinh',\n",
       " 'asinh_',\n",
       " 'atan',\n",
       " 'atan2',\n",
       " 'atan2_',\n",
       " 'atan_',\n",
       " 'atanh',\n",
       " 'atanh_',\n",
       " 'backward',\n",
       " 'baddbmm',\n",
       " 'baddbmm_',\n",
       " 'bernoulli',\n",
       " 'bernoulli_',\n",
       " 'bfloat16',\n",
       " 'bincount',\n",
       " 'bitwise_and',\n",
       " 'bitwise_and_',\n",
       " 'bitwise_not',\n",
       " 'bitwise_not_',\n",
       " 'bitwise_or',\n",
       " 'bitwise_or_',\n",
       " 'bitwise_xor',\n",
       " 'bitwise_xor_',\n",
       " 'bmm',\n",
       " 'bool',\n",
       " 'byte',\n",
       " 'cauchy_',\n",
       " 'ceil',\n",
       " 'ceil_',\n",
       " 'char',\n",
       " 'cholesky',\n",
       " 'cholesky_inverse',\n",
       " 'cholesky_solve',\n",
       " 'chunk',\n",
       " 'clamp',\n",
       " 'clamp_',\n",
       " 'clamp_max',\n",
       " 'clamp_max_',\n",
       " 'clamp_min',\n",
       " 'clamp_min_',\n",
       " 'clip',\n",
       " 'clip_',\n",
       " 'clone',\n",
       " 'coalesce',\n",
       " 'conj',\n",
       " 'contiguous',\n",
       " 'copy_',\n",
       " 'cos',\n",
       " 'cos_',\n",
       " 'cosh',\n",
       " 'cosh_',\n",
       " 'count_nonzero',\n",
       " 'cpu',\n",
       " 'cross',\n",
       " 'cuda',\n",
       " 'cummax',\n",
       " 'cummin',\n",
       " 'cumprod',\n",
       " 'cumsum',\n",
       " 'data',\n",
       " 'data_ptr',\n",
       " 'deg2rad',\n",
       " 'deg2rad_',\n",
       " 'dense_dim',\n",
       " 'dequantize',\n",
       " 'det',\n",
       " 'detach',\n",
       " 'detach_',\n",
       " 'device',\n",
       " 'diag',\n",
       " 'diag_embed',\n",
       " 'diagflat',\n",
       " 'diagonal',\n",
       " 'digamma',\n",
       " 'digamma_',\n",
       " 'dim',\n",
       " 'dist',\n",
       " 'div',\n",
       " 'div_',\n",
       " 'divide',\n",
       " 'divide_',\n",
       " 'dot',\n",
       " 'double',\n",
       " 'dtype',\n",
       " 'eig',\n",
       " 'element_size',\n",
       " 'eq',\n",
       " 'eq_',\n",
       " 'equal',\n",
       " 'erf',\n",
       " 'erf_',\n",
       " 'erfc',\n",
       " 'erfc_',\n",
       " 'erfinv',\n",
       " 'erfinv_',\n",
       " 'exp',\n",
       " 'exp2',\n",
       " 'exp2_',\n",
       " 'exp_',\n",
       " 'expand',\n",
       " 'expand_as',\n",
       " 'expm1',\n",
       " 'expm1_',\n",
       " 'exponential_',\n",
       " 'fft',\n",
       " 'fill_',\n",
       " 'fill_diagonal_',\n",
       " 'fix',\n",
       " 'fix_',\n",
       " 'flatten',\n",
       " 'flip',\n",
       " 'fliplr',\n",
       " 'flipud',\n",
       " 'float',\n",
       " 'floor',\n",
       " 'floor_',\n",
       " 'floor_divide',\n",
       " 'floor_divide_',\n",
       " 'fmod',\n",
       " 'fmod_',\n",
       " 'frac',\n",
       " 'frac_',\n",
       " 'gather',\n",
       " 'gcd',\n",
       " 'gcd_',\n",
       " 'ge',\n",
       " 'ge_',\n",
       " 'geometric_',\n",
       " 'geqrf',\n",
       " 'ger',\n",
       " 'get_device',\n",
       " 'grad',\n",
       " 'grad_fn',\n",
       " 'greater',\n",
       " 'greater_',\n",
       " 'greater_equal',\n",
       " 'greater_equal_',\n",
       " 'gt',\n",
       " 'gt_',\n",
       " 'half',\n",
       " 'hardshrink',\n",
       " 'has_names',\n",
       " 'heaviside',\n",
       " 'heaviside_',\n",
       " 'histc',\n",
       " 'hypot',\n",
       " 'hypot_',\n",
       " 'i0',\n",
       " 'i0_',\n",
       " 'ifft',\n",
       " 'imag',\n",
       " 'index_add',\n",
       " 'index_add_',\n",
       " 'index_copy',\n",
       " 'index_copy_',\n",
       " 'index_fill',\n",
       " 'index_fill_',\n",
       " 'index_put',\n",
       " 'index_put_',\n",
       " 'index_select',\n",
       " 'indices',\n",
       " 'int',\n",
       " 'int_repr',\n",
       " 'inverse',\n",
       " 'irfft',\n",
       " 'is_coalesced',\n",
       " 'is_complex',\n",
       " 'is_contiguous',\n",
       " 'is_cuda',\n",
       " 'is_distributed',\n",
       " 'is_floating_point',\n",
       " 'is_leaf',\n",
       " 'is_meta',\n",
       " 'is_mkldnn',\n",
       " 'is_nonzero',\n",
       " 'is_pinned',\n",
       " 'is_quantized',\n",
       " 'is_same_size',\n",
       " 'is_set_to',\n",
       " 'is_shared',\n",
       " 'is_signed',\n",
       " 'is_sparse',\n",
       " 'isclose',\n",
       " 'isfinite',\n",
       " 'isinf',\n",
       " 'isnan',\n",
       " 'isneginf',\n",
       " 'isposinf',\n",
       " 'isreal',\n",
       " 'istft',\n",
       " 'item',\n",
       " 'kthvalue',\n",
       " 'layout',\n",
       " 'lcm',\n",
       " 'lcm_',\n",
       " 'le',\n",
       " 'le_',\n",
       " 'lerp',\n",
       " 'lerp_',\n",
       " 'less',\n",
       " 'less_',\n",
       " 'less_equal',\n",
       " 'less_equal_',\n",
       " 'lgamma',\n",
       " 'lgamma_',\n",
       " 'log',\n",
       " 'log10',\n",
       " 'log10_',\n",
       " 'log1p',\n",
       " 'log1p_',\n",
       " 'log2',\n",
       " 'log2_',\n",
       " 'log_',\n",
       " 'log_normal_',\n",
       " 'log_softmax',\n",
       " 'logaddexp',\n",
       " 'logaddexp2',\n",
       " 'logcumsumexp',\n",
       " 'logdet',\n",
       " 'logical_and',\n",
       " 'logical_and_',\n",
       " 'logical_not',\n",
       " 'logical_not_',\n",
       " 'logical_or',\n",
       " 'logical_or_',\n",
       " 'logical_xor',\n",
       " 'logical_xor_',\n",
       " 'logit',\n",
       " 'logit_',\n",
       " 'logsumexp',\n",
       " 'long',\n",
       " 'lstsq',\n",
       " 'lt',\n",
       " 'lt_',\n",
       " 'lu',\n",
       " 'lu_solve',\n",
       " 'map2_',\n",
       " 'map_',\n",
       " 'masked_fill',\n",
       " 'masked_fill_',\n",
       " 'masked_scatter',\n",
       " 'masked_scatter_',\n",
       " 'masked_select',\n",
       " 'matmul',\n",
       " 'matrix_exp',\n",
       " 'matrix_power',\n",
       " 'max',\n",
       " 'maximum',\n",
       " 'mean',\n",
       " 'median',\n",
       " 'min',\n",
       " 'minimum',\n",
       " 'mm',\n",
       " 'mode',\n",
       " 'movedim',\n",
       " 'mul',\n",
       " 'mul_',\n",
       " 'multinomial',\n",
       " 'multiply',\n",
       " 'multiply_',\n",
       " 'mv',\n",
       " 'mvlgamma',\n",
       " 'mvlgamma_',\n",
       " 'name',\n",
       " 'names',\n",
       " 'nanquantile',\n",
       " 'nansum',\n",
       " 'narrow',\n",
       " 'narrow_copy',\n",
       " 'ndim',\n",
       " 'ndimension',\n",
       " 'ne',\n",
       " 'ne_',\n",
       " 'neg',\n",
       " 'neg_',\n",
       " 'negative',\n",
       " 'negative_',\n",
       " 'nelement',\n",
       " 'new',\n",
       " 'new_empty',\n",
       " 'new_full',\n",
       " 'new_ones',\n",
       " 'new_tensor',\n",
       " 'new_zeros',\n",
       " 'nextafter',\n",
       " 'nextafter_',\n",
       " 'nonzero',\n",
       " 'norm',\n",
       " 'normal_',\n",
       " 'not_equal',\n",
       " 'not_equal_',\n",
       " 'numel',\n",
       " 'numpy',\n",
       " 'orgqr',\n",
       " 'ormqr',\n",
       " 'outer',\n",
       " 'output_nr',\n",
       " 'permute',\n",
       " 'pin_memory',\n",
       " 'pinverse',\n",
       " 'polygamma',\n",
       " 'polygamma_',\n",
       " 'pow',\n",
       " 'pow_',\n",
       " 'prelu',\n",
       " 'prod',\n",
       " 'put_',\n",
       " 'q_per_channel_axis',\n",
       " 'q_per_channel_scales',\n",
       " 'q_per_channel_zero_points',\n",
       " 'q_scale',\n",
       " 'q_zero_point',\n",
       " 'qr',\n",
       " 'qscheme',\n",
       " 'quantile',\n",
       " 'rad2deg',\n",
       " 'rad2deg_',\n",
       " 'random_',\n",
       " 'real',\n",
       " 'reciprocal',\n",
       " 'reciprocal_',\n",
       " 'record_stream',\n",
       " 'refine_names',\n",
       " 'register_hook',\n",
       " 'reinforce',\n",
       " 'relu',\n",
       " 'relu_',\n",
       " 'remainder',\n",
       " 'remainder_',\n",
       " 'rename',\n",
       " 'rename_',\n",
       " 'renorm',\n",
       " 'renorm_',\n",
       " 'repeat',\n",
       " 'repeat_interleave',\n",
       " 'requires_grad',\n",
       " 'requires_grad_',\n",
       " 'reshape',\n",
       " 'reshape_as',\n",
       " 'resize',\n",
       " 'resize_',\n",
       " 'resize_as',\n",
       " 'resize_as_',\n",
       " 'retain_grad',\n",
       " 'rfft',\n",
       " 'roll',\n",
       " 'rot90',\n",
       " 'round',\n",
       " 'round_',\n",
       " 'rsqrt',\n",
       " 'rsqrt_',\n",
       " 'scatter',\n",
       " 'scatter_',\n",
       " 'scatter_add',\n",
       " 'scatter_add_',\n",
       " 'select',\n",
       " 'set_',\n",
       " 'sgn',\n",
       " 'sgn_',\n",
       " 'shape',\n",
       " 'share_memory_',\n",
       " 'short',\n",
       " 'sigmoid',\n",
       " 'sigmoid_',\n",
       " 'sign',\n",
       " 'sign_',\n",
       " 'signbit',\n",
       " 'sin',\n",
       " 'sin_',\n",
       " 'sinh',\n",
       " 'sinh_',\n",
       " 'size',\n",
       " 'slogdet',\n",
       " 'smm',\n",
       " 'softmax',\n",
       " 'solve',\n",
       " 'sort',\n",
       " 'sparse_dim',\n",
       " 'sparse_mask',\n",
       " 'sparse_resize_',\n",
       " 'sparse_resize_and_clear_',\n",
       " 'split',\n",
       " 'split_with_sizes',\n",
       " 'sqrt',\n",
       " 'sqrt_',\n",
       " 'square',\n",
       " 'square_',\n",
       " 'squeeze',\n",
       " 'squeeze_',\n",
       " 'sspaddmm',\n",
       " 'std',\n",
       " 'stft',\n",
       " 'storage',\n",
       " 'storage_offset',\n",
       " 'storage_type',\n",
       " 'stride',\n",
       " 'sub',\n",
       " 'sub_',\n",
       " 'subtract',\n",
       " 'subtract_',\n",
       " 'sum',\n",
       " 'sum_to_size',\n",
       " 'svd',\n",
       " 'symeig',\n",
       " 't',\n",
       " 't_',\n",
       " 'take',\n",
       " 'tan',\n",
       " 'tan_',\n",
       " 'tanh',\n",
       " 'tanh_',\n",
       " 'to',\n",
       " 'to_dense',\n",
       " 'to_mkldnn',\n",
       " 'to_sparse',\n",
       " 'tolist',\n",
       " 'topk',\n",
       " 'trace',\n",
       " 'transpose',\n",
       " 'transpose_',\n",
       " 'triangular_solve',\n",
       " 'tril',\n",
       " 'tril_',\n",
       " 'triu',\n",
       " 'triu_',\n",
       " 'true_divide',\n",
       " 'true_divide_',\n",
       " 'trunc',\n",
       " 'trunc_',\n",
       " 'type',\n",
       " 'type_as',\n",
       " 'unbind',\n",
       " 'unflatten',\n",
       " 'unfold',\n",
       " 'uniform_',\n",
       " 'unique',\n",
       " 'unique_consecutive',\n",
       " 'unsafe_chunk',\n",
       " 'unsafe_split',\n",
       " 'unsafe_split_with_sizes',\n",
       " 'unsqueeze',\n",
       " 'unsqueeze_',\n",
       " 'values',\n",
       " 'var',\n",
       " 'vdot',\n",
       " 'view',\n",
       " 'view_as',\n",
       " 'where',\n",
       " 'zero_']"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[o for o in dir(A) if not o.startswith(\"_\")]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# But what about the GPU?\n",
    "How do I use the GPU?\n",
    "\n",
    "If you have a GPU make sure that the right pytorch is installed\n",
    "(check https://pytorch.org/ for details)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.519401Z",
     "iopub.status.busy": "2020-11-27T14:29:58.518612Z",
     "iopub.status.idle": "2020-11-27T14:29:58.521013Z",
     "shell.execute_reply": "2020-11-27T14:29:58.521517Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you have a GPU you should get something like: \n",
    "`device(type='cuda', index=0)`\n",
    "\n",
    "You can move data to the GPU by doing `.to(device)`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:29:58.524698Z",
     "iopub.status.busy": "2020-11-27T14:29:58.524306Z",
     "iopub.status.idle": "2020-11-27T14:30:00.403525Z",
     "shell.execute_reply": "2020-11-27T14:30:00.403186Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 0., 0.],\n",
       "        [0., 1., 0.],\n",
       "        [0., 0., 1.]], device='cuda:0')"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = torch.eye(3)\n",
    "data = data.to(device)\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now the computation happens on the GPU."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.405997Z",
     "iopub.status.busy": "2020-11-27T14:30:00.405680Z",
     "iopub.status.idle": "2020-11-27T14:30:00.408950Z",
     "shell.execute_reply": "2020-11-27T14:30:00.408652Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[2., 0., 0.],\n",
       "        [0., 2., 0.],\n",
       "        [0., 0., 2.]], device='cuda:0')"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = data + data\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.411419Z",
     "iopub.status.busy": "2020-11-27T14:30:00.410815Z",
     "iopub.status.idle": "2020-11-27T14:30:00.413732Z",
     "shell.execute_reply": "2020-11-27T14:30:00.413445Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res.device"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: before `v0.4` one had to use `.cuda()` and `.cpu()` to move stuff to and from the GPU.\n",
    "This littered the code with many:\n",
    "```python\n",
    "if CUDA:\n",
    "    model = model.cuda()\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Automatic differentiation with `autograd`\n",
    "Prior to `v0.4` PyTorch used the class `Variable` to record gradients. You had to wrap `Tensor`s in `Variable`s.\n",
    "`Variable`s behaved exactly like `Tensors`.\n",
    "\n",
    "With `v0.4` `Tensor` can record gradients directly if you tell it do do so, e.g. `torch.ones(3, requires_grad=True)`.\n",
    "There is no need for `Variable` anymore.\n",
    "Many tutorials still use `Variable`, be aware!\n",
    "\n",
    "Ref:\n",
    "- https://pytorch.org/docs/stable/autograd.html\n",
    "- https://pytorch.org/tutorials/beginner/blitz/autograd_tutorial.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You rarely use `torch.autograd` directly.\n",
    "Pretty much everything is part or `torch.Tensor` now.\n",
    "Simply add `requires_grad=True` to the tensors you want to calculate the gradients for.\n",
    "`nn.Module` track gradients automatically."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.416379Z",
     "iopub.status.busy": "2020-11-27T14:30:00.416068Z",
     "iopub.status.idle": "2020-11-27T14:30:00.418236Z",
     "shell.execute_reply": "2020-11-27T14:30:00.417899Z"
    }
   },
   "outputs": [],
   "source": [
    "from torch import autograd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.420791Z",
     "iopub.status.busy": "2020-11-27T14:30:00.420372Z",
     "iopub.status.idle": "2020-11-27T14:30:00.423211Z",
     "shell.execute_reply": "2020-11-27T14:30:00.422811Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(2.)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.tensor(2.)\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.425473Z",
     "iopub.status.busy": "2020-11-27T14:30:00.425104Z",
     "iopub.status.idle": "2020-11-27T14:30:00.428200Z",
     "shell.execute_reply": "2020-11-27T14:30:00.427797Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(2., requires_grad=True)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.tensor(2., requires_grad=True)\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.431055Z",
     "iopub.status.busy": "2020-11-27T14:30:00.430559Z",
     "iopub.status.idle": "2020-11-27T14:30:00.433053Z",
     "shell.execute_reply": "2020-11-27T14:30:00.432651Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "print(x.requires_grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.435878Z",
     "iopub.status.busy": "2020-11-27T14:30:00.435424Z",
     "iopub.status.idle": "2020-11-27T14:30:00.437829Z",
     "shell.execute_reply": "2020-11-27T14:30:00.437427Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n"
     ]
    }
   ],
   "source": [
    "print(x.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.440932Z",
     "iopub.status.busy": "2020-11-27T14:30:00.440436Z",
     "iopub.status.idle": "2020-11-27T14:30:00.442608Z",
     "shell.execute_reply": "2020-11-27T14:30:00.442946Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Grad of x: None\n"
     ]
    }
   ],
   "source": [
    "y = x ** 2\n",
    "\n",
    "print(\"Grad of x:\", x.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.445966Z",
     "iopub.status.busy": "2020-11-27T14:30:00.445404Z",
     "iopub.status.idle": "2020-11-27T14:30:00.448552Z",
     "shell.execute_reply": "2020-11-27T14:30:00.448152Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Grad of x: tensor(4.)\n"
     ]
    }
   ],
   "source": [
    "y = x ** 2\n",
    "y.backward()\n",
    "\n",
    "print(\"Grad of x:\", x.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.450955Z",
     "iopub.status.busy": "2020-11-27T14:30:00.450552Z",
     "iopub.status.idle": "2020-11-27T14:30:00.452588Z",
     "shell.execute_reply": "2020-11-27T14:30:00.452191Z"
    }
   },
   "outputs": [],
   "source": [
    "# What is going to happen here?\n",
    "# x = torch.tensor(2.)\n",
    "# x.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.455926Z",
     "iopub.status.busy": "2020-11-27T14:30:00.455467Z",
     "iopub.status.idle": "2020-11-27T14:30:00.457873Z",
     "shell.execute_reply": "2020-11-27T14:30:00.457473Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n"
     ]
    }
   ],
   "source": [
    "# Don't record the gradient\n",
    "# Useful for inference\n",
    "\n",
    "params = torch.tensor(2., requires_grad=True)\n",
    "\n",
    "with torch.no_grad():\n",
    "    y = x * x\n",
    "    print(x.grad_fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`nn.Module` and `nn.Parameter` keep track of gradients for you."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.461110Z",
     "iopub.status.busy": "2020-11-27T14:30:00.460615Z",
     "iopub.status.idle": "2020-11-27T14:30:00.463220Z",
     "shell.execute_reply": "2020-11-27T14:30:00.463548Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[-0.3140, -0.0996]], requires_grad=True)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lin = nn.Linear(2, 1, bias=True)\n",
    "lin.weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.466457Z",
     "iopub.status.busy": "2020-11-27T14:30:00.465919Z",
     "iopub.status.idle": "2020-11-27T14:30:00.468095Z",
     "shell.execute_reply": "2020-11-27T14:30:00.468422Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.nn.parameter.Parameter"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(lin.weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.471392Z",
     "iopub.status.busy": "2020-11-27T14:30:00.470854Z",
     "iopub.status.idle": "2020-11-27T14:30:00.473012Z",
     "shell.execute_reply": "2020-11-27T14:30:00.473337Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "isinstance(lin.weight, torch.FloatTensor)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `torch.nn`\n",
    "The neural network modules contains many different layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.476007Z",
     "iopub.status.busy": "2020-11-27T14:30:00.475614Z",
     "iopub.status.idle": "2020-11-27T14:30:00.477622Z",
     "shell.execute_reply": "2020-11-27T14:30:00.477221Z"
    }
   },
   "outputs": [],
   "source": [
    "from torch import nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.480846Z",
     "iopub.status.busy": "2020-11-27T14:30:00.480310Z",
     "iopub.status.idle": "2020-11-27T14:30:00.482495Z",
     "shell.execute_reply": "2020-11-27T14:30:00.482801Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Linear(in_features=1, out_features=1, bias=True)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lin_reg = nn.Linear(1, 1, bias=True)\n",
    "lin_reg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.485645Z",
     "iopub.status.busy": "2020-11-27T14:30:00.485113Z",
     "iopub.status.idle": "2020-11-27T14:30:00.487277Z",
     "shell.execute_reply": "2020-11-27T14:30:00.487600Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.nn.modules.conv.Conv2d"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.Conv2d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.490470Z",
     "iopub.status.busy": "2020-11-27T14:30:00.489935Z",
     "iopub.status.idle": "2020-11-27T14:30:00.492595Z",
     "shell.execute_reply": "2020-11-27T14:30:00.492194Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.nn.modules.conv.Conv3d"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.Conv3d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.495476Z",
     "iopub.status.busy": "2020-11-27T14:30:00.494941Z",
     "iopub.status.idle": "2020-11-27T14:30:00.497272Z",
     "shell.execute_reply": "2020-11-27T14:30:00.497581Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.nn.modules.batchnorm.BatchNorm2d"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.BatchNorm2d"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Activations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.500353Z",
     "iopub.status.busy": "2020-11-27T14:30:00.499871Z",
     "iopub.status.idle": "2020-11-27T14:30:00.501961Z",
     "shell.execute_reply": "2020-11-27T14:30:00.502252Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.nn.modules.activation.ReLU"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.ReLU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.505300Z",
     "iopub.status.busy": "2020-11-27T14:30:00.504477Z",
     "iopub.status.idle": "2020-11-27T14:30:00.507796Z",
     "shell.execute_reply": "2020-11-27T14:30:00.507411Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.nn.modules.activation.Sigmoid"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.Sigmoid"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Losses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.510555Z",
     "iopub.status.busy": "2020-11-27T14:30:00.510049Z",
     "iopub.status.idle": "2020-11-27T14:30:00.512727Z",
     "shell.execute_reply": "2020-11-27T14:30:00.512338Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.nn.modules.activation.Softmax"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.Softmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.515550Z",
     "iopub.status.busy": "2020-11-27T14:30:00.514975Z",
     "iopub.status.idle": "2020-11-27T14:30:00.518208Z",
     "shell.execute_reply": "2020-11-27T14:30:00.517831Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.nn.modules.loss.CrossEntropyLoss"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.CrossEntropyLoss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.521352Z",
     "iopub.status.busy": "2020-11-27T14:30:00.520650Z",
     "iopub.status.idle": "2020-11-27T14:30:00.522777Z",
     "shell.execute_reply": "2020-11-27T14:30:00.523111Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.nn.modules.loss.BCELoss"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.BCELoss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.525577Z",
     "iopub.status.busy": "2020-11-27T14:30:00.525078Z",
     "iopub.status.idle": "2020-11-27T14:30:00.527201Z",
     "shell.execute_reply": "2020-11-27T14:30:00.527540Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.nn.modules.loss.MSELoss"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.MSELoss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Functional (stateless) alternatives"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.530500Z",
     "iopub.status.busy": "2020-11-27T14:30:00.530162Z",
     "iopub.status.idle": "2020-11-27T14:30:00.532369Z",
     "shell.execute_reply": "2020-11-27T14:30:00.531976Z"
    }
   },
   "outputs": [],
   "source": [
    "from torch.nn import functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.535350Z",
     "iopub.status.busy": "2020-11-27T14:30:00.534785Z",
     "iopub.status.idle": "2020-11-27T14:30:00.536984Z",
     "shell.execute_reply": "2020-11-27T14:30:00.537267Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<function torch.nn.functional.mse_loss(input, target, size_average=None, reduce=None, reduction='mean')>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "F.mse_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.540054Z",
     "iopub.status.busy": "2020-11-27T14:30:00.539484Z",
     "iopub.status.idle": "2020-11-27T14:30:00.542063Z",
     "shell.execute_reply": "2020-11-27T14:30:00.541619Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<function torch.nn.functional.relu(input: torch.Tensor, inplace: bool = False) -> torch.Tensor>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "F.relu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.544823Z",
     "iopub.status.busy": "2020-11-27T14:30:00.544278Z",
     "iopub.status.idle": "2020-11-27T14:30:00.547247Z",
     "shell.execute_reply": "2020-11-27T14:30:00.546803Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<function torch.nn.functional.relu6(input, inplace=False)>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "F.relu6"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `torch.optim`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.549599Z",
     "iopub.status.busy": "2020-11-27T14:30:00.549259Z",
     "iopub.status.idle": "2020-11-27T14:30:00.551396Z",
     "shell.execute_reply": "2020-11-27T14:30:00.551048Z"
    }
   },
   "outputs": [],
   "source": [
    "from torch import optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.554126Z",
     "iopub.status.busy": "2020-11-27T14:30:00.553539Z",
     "iopub.status.idle": "2020-11-27T14:30:00.555775Z",
     "shell.execute_reply": "2020-11-27T14:30:00.556049Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.optim.sgd.SGD"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optim.SGD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.558935Z",
     "iopub.status.busy": "2020-11-27T14:30:00.558419Z",
     "iopub.status.idle": "2020-11-27T14:30:00.561482Z",
     "shell.execute_reply": "2020-11-27T14:30:00.561106Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.optim.adam.Adam"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optim.Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-27T14:30:00.564333Z",
     "iopub.status.busy": "2020-11-27T14:30:00.563977Z",
     "iopub.status.idle": "2020-11-27T14:30:00.566649Z",
     "shell.execute_reply": "2020-11-27T14:30:00.566334Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.optim.adamw.AdamW"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optim.AdamW"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercise\n",
    "- Do you remember the analytical solution to solve for the parameters of linear regression? Implement it."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "toc-autonumbering": true
 },
 "nbformat": 4,
 "nbformat_minor": 4
}