notebooks/pytorch_basics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc-hr-collapsed": true
   },
   "source": [
    "# PyTorch Basics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Init, helpers, utils, ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torchvision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pprint import pprint\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from IPython.core.debugger import set_trace"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "toc-hr-collapsed": true
   },
   "source": [
    "# Tensors\n",
    "tensors - the atoms of machine learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tensors in numpy and pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from numpy.linalg import inv\n",
    "from numpy.linalg import multi_dot as mdot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 0., 0.],\n",
       "       [0., 1., 0.],\n",
       "       [0., 0., 1.]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# numpy\n",
    "np.eye(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 0., 0.],\n",
       "        [0., 1., 0.],\n",
       "        [0., 0., 1.]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# torch\n",
    "torch.eye(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.00312978, 0.165447  , 0.03888296],\n",
       "       [0.60600685, 0.59181157, 0.75366668],\n",
       "       [0.45594334, 0.02011842, 0.87961916],\n",
       "       [0.05319178, 0.8845105 , 0.61772618],\n",
       "       [0.07897388, 0.31597133, 0.69900588]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# numpy\n",
    "X = np.random.random((5, 3))\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.7300, 0.7018, 0.2601],\n",
       "        [0.1506, 0.4892, 0.9874],\n",
       "        [0.9033, 0.3847, 0.9157],\n",
       "        [0.7148, 0.9781, 0.9281],\n",
       "        [0.9621, 0.1599, 0.4295]])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# pytorch\n",
    "Y = torch.rand((5, 3))\n",
    "Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 3)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([5, 3])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.58420467, 0.44033471, 0.94596653],\n",
       "       [0.44033471, 1.26021511, 1.23740939],\n",
       "       [0.94596653, 1.23740939, 2.21345008]])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# numpy\n",
    "X.T @ X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[2.8080, 1.7865, 2.2423],\n",
       "        [1.7865, 1.8621, 1.9942],\n",
       "        [2.2423, 1.9942, 2.9268]])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# torch\n",
    "Y.t() @ Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 6.01876353,  0.93702967, -3.09608886],\n",
       "       [ 0.93702967,  1.90504317, -1.4654575 ],\n",
       "       [-3.09608886, -1.4654575 ,  2.59421586]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# numpy\n",
    "inv(X.T @ X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 1.0446, -0.5369, -0.4345],\n",
       "        [-0.5369,  2.2630, -1.1306],\n",
       "        [-0.4345, -1.1306,  1.4449]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# torch\n",
    "torch.inverse(Y.t() @ Y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## More on PyTorch Tensors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Operations are also available as methods."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[2., 1., 1.],\n",
       "        [1., 2., 1.],\n",
       "        [1., 1., 2.]])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A = torch.eye(3)\n",
    "A.add(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 0., 0.],\n",
       "        [0., 1., 0.],\n",
       "        [0., 0., 1.]])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Any operation that mutates a tensor in-place has a `_` suffix."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[2., 1., 1.],\n",
       "        [1., 2., 1.],\n",
       "        [1., 1., 2.]])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A.add_(1)\n",
    "A"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Indexing and broadcasting\n",
    "It works as expected/like numpy:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(2.)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A[0, 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([2., 1., 1.])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[2., 1., 1.],\n",
       "        [1., 2., 1.]])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A[0:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 1.],\n",
       "        [2., 1.],\n",
       "        [1., 2.]])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A[:, 1:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Converting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 0., 0.],\n",
       "        [0., 1., 0.],\n",
       "        [0., 0., 1.]])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A = torch.eye(3)\n",
    "A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 0., 0.],\n",
       "       [0., 1., 0.],\n",
       "       [0., 0., 1.]], dtype=float32)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# torch --> numpy\n",
    "B = A.numpy()\n",
    "B"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: torch and numpy can share the same memory / zero-copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1.5000, 0.5000, 0.5000],\n",
       "        [0.5000, 1.5000, 0.5000],\n",
       "        [0.5000, 0.5000, 1.5000]])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A.add_(.5)\n",
    "A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.5, 0.5, 0.5],\n",
       "       [0.5, 1.5, 0.5],\n",
       "       [0.5, 0.5, 1.5]], dtype=float32)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "B"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 0., 0.],\n",
       "        [0., 1., 0.],\n",
       "        [0., 0., 1.]], dtype=torch.float64)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# numpy --> torch\n",
    "torch.from_numpy(np.eye(3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Much more"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['AVG',\n",
       " 'AggregationType',\n",
       " 'Argument',\n",
       " 'ArgumentSpec',\n",
       " 'Block',\n",
       " 'BoolStorage',\n",
       " 'BoolTensor',\n",
       " 'BoolType',\n",
       " 'ByteStorage',\n",
       " 'ByteTensor',\n",
       " 'CharStorage',\n",
       " 'CharTensor',\n",
       " 'Code',\n",
       " 'CompilationUnit',\n",
       " 'CompleteArgumentSpec',\n",
       " 'DictType',\n",
       " 'DoubleStorage',\n",
       " 'DoubleTensor',\n",
       " 'ExecutionPlanState',\n",
       " 'ExtraFilesMap',\n",
       " 'FatalError',\n",
       " 'FileCheck',\n",
       " 'FloatStorage',\n",
       " 'FloatTensor',\n",
       " 'FloatType',\n",
       " 'Function',\n",
       " 'FunctionSchema',\n",
       " 'Future',\n",
       " 'Generator',\n",
       " 'Gradient',\n",
       " 'Graph',\n",
       " 'GraphExecutorState',\n",
       " 'HalfStorage',\n",
       " 'HalfStorageBase',\n",
       " 'HalfTensor',\n",
       " 'IODescriptor',\n",
       " 'IntStorage',\n",
       " 'IntTensor',\n",
       " 'IntType',\n",
       " 'JITException',\n",
       " 'ListType',\n",
       " 'LockingLogger',\n",
       " 'LoggerBase',\n",
       " 'LongStorage',\n",
       " 'LongTensor',\n",
       " 'Node',\n",
       " 'NoopLogger',\n",
       " 'NumberType',\n",
       " 'OptionalType',\n",
       " 'PyTorchFileReader',\n",
       " 'PyTorchFileWriter',\n",
       " 'SUM',\n",
       " 'ScriptMethod',\n",
       " 'ScriptModule',\n",
       " 'ShortStorage',\n",
       " 'ShortTensor',\n",
       " 'Size',\n",
       " 'Storage',\n",
       " 'StringType',\n",
       " 'Tensor',\n",
       " 'TensorType',\n",
       " 'TracingState',\n",
       " 'TupleType',\n",
       " 'Type',\n",
       " 'Use',\n",
       " 'Value',\n",
       " 'abs',\n",
       " 'abs_',\n",
       " 'acos',\n",
       " 'acos_',\n",
       " 'adaptive_avg_pool1d',\n",
       " 'adaptive_max_pool1d',\n",
       " 'add',\n",
       " 'addbmm',\n",
       " 'addcdiv',\n",
       " 'addcmul',\n",
       " 'addmm',\n",
       " 'addmv',\n",
       " 'addmv_',\n",
       " 'addr',\n",
       " 'affine_grid_generator',\n",
       " 'all',\n",
       " 'allclose',\n",
       " 'alpha_dropout',\n",
       " 'alpha_dropout_',\n",
       " 'any',\n",
       " 'arange',\n",
       " 'argmax',\n",
       " 'argmin',\n",
       " 'argsort',\n",
       " 'as_strided',\n",
       " 'as_strided_',\n",
       " 'as_tensor',\n",
       " 'asin',\n",
       " 'asin_',\n",
       " 'atan',\n",
       " 'atan2',\n",
       " 'atan_',\n",
       " 'autograd',\n",
       " 'avg_pool1d',\n",
       " 'backends',\n",
       " 'baddbmm',\n",
       " 'bartlett_window',\n",
       " 'batch_norm',\n",
       " 'batch_norm_backward_elemt',\n",
       " 'batch_norm_backward_reduce',\n",
       " 'batch_norm_elemt',\n",
       " 'batch_norm_gather_stats',\n",
       " 'batch_norm_stats',\n",
       " 'batch_norm_update_stats',\n",
       " 'bernoulli',\n",
       " 'bilinear',\n",
       " 'binary_cross_entropy_with_logits',\n",
       " 'bincount',\n",
       " 'blackman_window',\n",
       " 'bmm',\n",
       " 'bool',\n",
       " 'broadcast_tensors',\n",
       " 'btrifact',\n",
       " 'btrifact_with_info',\n",
       " 'btrisolve',\n",
       " 'btriunpack',\n",
       " 'cartesian_prod',\n",
       " 'cat',\n",
       " 'cdist',\n",
       " 'ceil',\n",
       " 'ceil_',\n",
       " 'celu',\n",
       " 'celu_',\n",
       " 'chain_matmul',\n",
       " 'cholesky',\n",
       " 'cholesky_inverse',\n",
       " 'cholesky_solve',\n",
       " 'chunk',\n",
       " 'clamp',\n",
       " 'clamp_',\n",
       " 'clamp_max',\n",
       " 'clamp_max_',\n",
       " 'clamp_min',\n",
       " 'clamp_min_',\n",
       " 'clone',\n",
       " 'combinations',\n",
       " 'compiled_with_cxx11_abi',\n",
       " 'complex128',\n",
       " 'complex32',\n",
       " 'complex64',\n",
       " 'constant_pad_nd',\n",
       " 'conv1d',\n",
       " 'conv2d',\n",
       " 'conv3d',\n",
       " 'conv_tbc',\n",
       " 'conv_transpose1d',\n",
       " 'conv_transpose2d',\n",
       " 'conv_transpose3d',\n",
       " 'convolution',\n",
       " 'cos',\n",
       " 'cos_',\n",
       " 'cosh',\n",
       " 'cosh_',\n",
       " 'cosine_embedding_loss',\n",
       " 'cosine_similarity',\n",
       " 'cpp',\n",
       " 'cross',\n",
       " 'ctc_loss',\n",
       " 'cuda',\n",
       " 'cudnn_affine_grid_generator',\n",
       " 'cudnn_batch_norm',\n",
       " 'cudnn_convolution',\n",
       " 'cudnn_convolution_transpose',\n",
       " 'cudnn_grid_sampler',\n",
       " 'cudnn_is_acceptable',\n",
       " 'cumprod',\n",
       " 'cumsum',\n",
       " 'default_generator',\n",
       " 'dequantize',\n",
       " 'det',\n",
       " 'detach',\n",
       " 'detach_',\n",
       " 'device',\n",
       " 'diag',\n",
       " 'diag_embed',\n",
       " 'diagflat',\n",
       " 'diagonal',\n",
       " 'digamma',\n",
       " 'dist',\n",
       " 'distributed',\n",
       " 'distributions',\n",
       " 'div',\n",
       " 'dot',\n",
       " 'double',\n",
       " 'dropout',\n",
       " 'dropout_',\n",
       " 'dsmm',\n",
       " 'dtype',\n",
       " 'eig',\n",
       " 'einsum',\n",
       " 'embedding',\n",
       " 'embedding_bag',\n",
       " 'embedding_renorm_',\n",
       " 'empty',\n",
       " 'empty_like',\n",
       " 'empty_strided',\n",
       " 'enable_grad',\n",
       " 'eq',\n",
       " 'equal',\n",
       " 'erf',\n",
       " 'erf_',\n",
       " 'erfc',\n",
       " 'erfc_',\n",
       " 'erfinv',\n",
       " 'exp',\n",
       " 'exp_',\n",
       " 'expm1',\n",
       " 'expm1_',\n",
       " 'eye',\n",
       " 'fbgemm_is_cpu_supported',\n",
       " 'fbgemm_linear_int8_weight',\n",
       " 'fbgemm_linear_quantize_weight',\n",
       " 'fbgemm_pack_quantized_matrix',\n",
       " 'feature_alpha_dropout',\n",
       " 'feature_alpha_dropout_',\n",
       " 'feature_dropout',\n",
       " 'feature_dropout_',\n",
       " 'fft',\n",
       " 'fill_',\n",
       " 'finfo',\n",
       " 'flatten',\n",
       " 'flip',\n",
       " 'float',\n",
       " 'float16',\n",
       " 'float32',\n",
       " 'float64',\n",
       " 'floor',\n",
       " 'floor_',\n",
       " 'fmod',\n",
       " 'fork',\n",
       " 'frac',\n",
       " 'frac_',\n",
       " 'frobenius_norm',\n",
       " 'from_file',\n",
       " 'from_numpy',\n",
       " 'full',\n",
       " 'full_like',\n",
       " 'functional',\n",
       " 'gather',\n",
       " 'ge',\n",
       " 'gels',\n",
       " 'geqrf',\n",
       " 'ger',\n",
       " 'gesv',\n",
       " 'get_default_dtype',\n",
       " 'get_device',\n",
       " 'get_file_path',\n",
       " 'get_num_threads',\n",
       " 'get_rng_state',\n",
       " 'grid_sampler',\n",
       " 'grid_sampler_2d',\n",
       " 'grid_sampler_3d',\n",
       " 'group_norm',\n",
       " 'gru',\n",
       " 'gru_cell',\n",
       " 'gt',\n",
       " 'half',\n",
       " 'hamming_window',\n",
       " 'hann_window',\n",
       " 'hardshrink',\n",
       " 'has_cuda',\n",
       " 'has_cudnn',\n",
       " 'has_lapack',\n",
       " 'has_mkl',\n",
       " 'has_mkldnn',\n",
       " 'has_openmp',\n",
       " 'hinge_embedding_loss',\n",
       " 'histc',\n",
       " 'hsmm',\n",
       " 'hspmm',\n",
       " 'hub',\n",
       " 'ifft',\n",
       " 'iinfo',\n",
       " 'import_ir_module',\n",
       " 'import_ir_module_from_buffer',\n",
       " 'index_add',\n",
       " 'index_copy',\n",
       " 'index_fill',\n",
       " 'index_put',\n",
       " 'index_put_',\n",
       " 'index_select',\n",
       " 'initial_seed',\n",
       " 'instance_norm',\n",
       " 'int',\n",
       " 'int16',\n",
       " 'int32',\n",
       " 'int64',\n",
       " 'int8',\n",
       " 'int_repr',\n",
       " 'inverse',\n",
       " 'irfft',\n",
       " 'is_anomaly_enabled',\n",
       " 'is_complex',\n",
       " 'is_distributed',\n",
       " 'is_floating_point',\n",
       " 'is_grad_enabled',\n",
       " 'is_nonzero',\n",
       " 'is_same_size',\n",
       " 'is_signed',\n",
       " 'is_storage',\n",
       " 'is_tensor',\n",
       " 'isclose',\n",
       " 'isfinite',\n",
       " 'isinf',\n",
       " 'isnan',\n",
       " 'jit',\n",
       " 'kl_div',\n",
       " 'kthvalue',\n",
       " 'layer_norm',\n",
       " 'layout',\n",
       " 'le',\n",
       " 'lerp',\n",
       " 'lgamma',\n",
       " 'linspace',\n",
       " 'load',\n",
       " 'log',\n",
       " 'log10',\n",
       " 'log10_',\n",
       " 'log1p',\n",
       " 'log1p_',\n",
       " 'log2',\n",
       " 'log2_',\n",
       " 'log_',\n",
       " 'log_softmax',\n",
       " 'logdet',\n",
       " 'logspace',\n",
       " 'logsumexp',\n",
       " 'long',\n",
       " 'lstm',\n",
       " 'lstm_cell',\n",
       " 'lt',\n",
       " 'lu',\n",
       " 'lu_solve',\n",
       " 'lu_unpack',\n",
       " 'manual_seed',\n",
       " 'margin_ranking_loss',\n",
       " 'masked_fill',\n",
       " 'masked_scatter',\n",
       " 'masked_select',\n",
       " 'matmul',\n",
       " 'matrix_power',\n",
       " 'matrix_rank',\n",
       " 'max',\n",
       " 'max_pool1d',\n",
       " 'max_pool1d_with_indices',\n",
       " 'max_pool2d',\n",
       " 'max_pool3d',\n",
       " 'mean',\n",
       " 'median',\n",
       " 'merge_type_from_type_comment',\n",
       " 'meshgrid',\n",
       " 'min',\n",
       " 'miopen_batch_norm',\n",
       " 'miopen_convolution',\n",
       " 'miopen_convolution_transpose',\n",
       " 'miopen_depthwise_convolution',\n",
       " 'mkldnn_convolution',\n",
       " 'mkldnn_convolution_backward_weights',\n",
       " 'mkldnn_max_pool2d',\n",
       " 'mkldnn_reshape',\n",
       " 'mm',\n",
       " 'mode',\n",
       " 'mul',\n",
       " 'multinomial',\n",
       " 'multiprocessing',\n",
       " 'mv',\n",
       " 'mvlgamma',\n",
       " 'name',\n",
       " 'narrow',\n",
       " 'native_batch_norm',\n",
       " 'native_norm',\n",
       " 'ne',\n",
       " 'neg',\n",
       " 'neg_',\n",
       " 'nn',\n",
       " 'no_grad',\n",
       " 'nonzero',\n",
       " 'norm',\n",
       " 'norm_except_dim',\n",
       " 'normal',\n",
       " 'nuclear_norm',\n",
       " 'numel',\n",
       " 'ones',\n",
       " 'ones_like',\n",
       " 'onnx',\n",
       " 'ops',\n",
       " 'optim',\n",
       " 'orgqr',\n",
       " 'ormqr',\n",
       " 'os',\n",
       " 'pairwise_distance',\n",
       " 'parse_ir',\n",
       " 'parse_type_comment',\n",
       " 'pdist',\n",
       " 'pin_memory',\n",
       " 'pinverse',\n",
       " 'pixel_shuffle',\n",
       " 'platform',\n",
       " 'poisson',\n",
       " 'polygamma',\n",
       " 'potrf',\n",
       " 'potri',\n",
       " 'potrs',\n",
       " 'pow',\n",
       " 'prelu',\n",
       " 'prepare_multiprocessing_environment',\n",
       " 'prod',\n",
       " 'pstrf',\n",
       " 'q_scale',\n",
       " 'q_zero_point',\n",
       " 'qint8',\n",
       " 'qr',\n",
       " 'quantize_linear',\n",
       " 'quantized_gru_cell',\n",
       " 'quantized_lstm',\n",
       " 'quantized_lstm_cell',\n",
       " 'quantized_rnn_relu_cell',\n",
       " 'quantized_rnn_tanh_cell',\n",
       " 'quasirandom',\n",
       " 'rand',\n",
       " 'rand_like',\n",
       " 'randint',\n",
       " 'randint_like',\n",
       " 'randn',\n",
       " 'randn_like',\n",
       " 'random',\n",
       " 'randperm',\n",
       " 'range',\n",
       " 'reciprocal',\n",
       " 'reciprocal_',\n",
       " 'relu',\n",
       " 'relu_',\n",
       " 'remainder',\n",
       " 'renorm',\n",
       " 'repeat_interleave',\n",
       " 'reshape',\n",
       " 'resize_as_',\n",
       " 'rfft',\n",
       " 'rnn_relu',\n",
       " 'rnn_relu_cell',\n",
       " 'rnn_tanh',\n",
       " 'rnn_tanh_cell',\n",
       " 'roll',\n",
       " 'rot90',\n",
       " 'round',\n",
       " 'round_',\n",
       " 'rrelu',\n",
       " 'rrelu_',\n",
       " 'rsqrt',\n",
       " 'rsqrt_',\n",
       " 'rsub',\n",
       " 's_copy_',\n",
       " 's_native_addmm',\n",
       " 's_native_addmm_',\n",
       " 'saddmm',\n",
       " 'save',\n",
       " 'scalar_tensor',\n",
       " 'scatter',\n",
       " 'scatter_add',\n",
       " 'select',\n",
       " 'selu',\n",
       " 'selu_',\n",
       " 'serialization',\n",
       " 'set_anomaly_enabled',\n",
       " 'set_default_dtype',\n",
       " 'set_default_tensor_type',\n",
       " 'set_flush_denormal',\n",
       " 'set_grad_enabled',\n",
       " 'set_num_threads',\n",
       " 'set_printoptions',\n",
       " 'set_rng_state',\n",
       " 'short',\n",
       " 'sigmoid',\n",
       " 'sigmoid_',\n",
       " 'sign',\n",
       " 'sin',\n",
       " 'sin_',\n",
       " 'sinh',\n",
       " 'sinh_',\n",
       " 'slogdet',\n",
       " 'smm',\n",
       " 'softmax',\n",
       " 'solve',\n",
       " 'sort',\n",
       " 'sparse',\n",
       " 'sparse_coo',\n",
       " 'sparse_coo_tensor',\n",
       " 'split',\n",
       " 'split_with_sizes',\n",
       " 'spmm',\n",
       " 'sqrt',\n",
       " 'sqrt_',\n",
       " 'squeeze',\n",
       " 'sspaddmm',\n",
       " 'stack',\n",
       " 'std',\n",
       " 'stft',\n",
       " 'storage',\n",
       " 'strided',\n",
       " 'sub',\n",
       " 'sum',\n",
       " 'svd',\n",
       " 'symeig',\n",
       " 'sys',\n",
       " 't',\n",
       " 'take',\n",
       " 'tan',\n",
       " 'tan_',\n",
       " 'tanh',\n",
       " 'tanh_',\n",
       " 'tensor',\n",
       " 'tensordot',\n",
       " 'testing',\n",
       " 'threshold',\n",
       " 'threshold_',\n",
       " 'topk',\n",
       " 'torch',\n",
       " 'trace',\n",
       " 'transpose',\n",
       " 'triangular_solve',\n",
       " 'tril',\n",
       " 'tril_indices',\n",
       " 'triplet_margin_loss',\n",
       " 'triu',\n",
       " 'triu_indices',\n",
       " 'trtrs',\n",
       " 'trunc',\n",
       " 'trunc_',\n",
       " 'typename',\n",
       " 'uint8',\n",
       " 'unbind',\n",
       " 'unique',\n",
       " 'unique_consecutive',\n",
       " 'unsqueeze',\n",
       " 'utils',\n",
       " 'var',\n",
       " 'version',\n",
       " 'wait',\n",
       " 'where',\n",
       " 'zero_',\n",
       " 'zeros',\n",
       " 'zeros_like']"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[o for o  in dir(torch) if not o.startswith(\"_\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['abs',\n",
       " 'abs_',\n",
       " 'acos',\n",
       " 'acos_',\n",
       " 'add',\n",
       " 'add_',\n",
       " 'addbmm',\n",
       " 'addbmm_',\n",
       " 'addcdiv',\n",
       " 'addcdiv_',\n",
       " 'addcmul',\n",
       " 'addcmul_',\n",
       " 'addmm',\n",
       " 'addmm_',\n",
       " 'addmv',\n",
       " 'addmv_',\n",
       " 'addr',\n",
       " 'addr_',\n",
       " 'all',\n",
       " 'allclose',\n",
       " 'any',\n",
       " 'apply_',\n",
       " 'argmax',\n",
       " 'argmin',\n",
       " 'argsort',\n",
       " 'as_strided',\n",
       " 'as_strided_',\n",
       " 'asin',\n",
       " 'asin_',\n",
       " 'atan',\n",
       " 'atan2',\n",
       " 'atan2_',\n",
       " 'atan_',\n",
       " 'backward',\n",
       " 'baddbmm',\n",
       " 'baddbmm_',\n",
       " 'bernoulli',\n",
       " 'bernoulli_',\n",
       " 'bincount',\n",
       " 'bmm',\n",
       " 'btrifact',\n",
       " 'btrifact_with_info',\n",
       " 'btrisolve',\n",
       " 'byte',\n",
       " 'cauchy_',\n",
       " 'ceil',\n",
       " 'ceil_',\n",
       " 'char',\n",
       " 'cholesky',\n",
       " 'cholesky_inverse',\n",
       " 'cholesky_solve',\n",
       " 'chunk',\n",
       " 'clamp',\n",
       " 'clamp_',\n",
       " 'clamp_max',\n",
       " 'clamp_max_',\n",
       " 'clamp_min',\n",
       " 'clamp_min_',\n",
       " 'clone',\n",
       " 'coalesce',\n",
       " 'contiguous',\n",
       " 'copy_',\n",
       " 'cos',\n",
       " 'cos_',\n",
       " 'cosh',\n",
       " 'cosh_',\n",
       " 'cpu',\n",
       " 'cross',\n",
       " 'cuda',\n",
       " 'cumprod',\n",
       " 'cumsum',\n",
       " 'data',\n",
       " 'data_ptr',\n",
       " 'dense_dim',\n",
       " 'dequantize',\n",
       " 'det',\n",
       " 'detach',\n",
       " 'detach_',\n",
       " 'device',\n",
       " 'diag',\n",
       " 'diag_embed',\n",
       " 'diagflat',\n",
       " 'diagonal',\n",
       " 'digamma',\n",
       " 'digamma_',\n",
       " 'dim',\n",
       " 'dist',\n",
       " 'div',\n",
       " 'div_',\n",
       " 'dot',\n",
       " 'double',\n",
       " 'dtype',\n",
       " 'eig',\n",
       " 'element_size',\n",
       " 'eq',\n",
       " 'eq_',\n",
       " 'equal',\n",
       " 'erf',\n",
       " 'erf_',\n",
       " 'erfc',\n",
       " 'erfc_',\n",
       " 'erfinv',\n",
       " 'erfinv_',\n",
       " 'exp',\n",
       " 'exp_',\n",
       " 'expand',\n",
       " 'expand_as',\n",
       " 'expm1',\n",
       " 'expm1_',\n",
       " 'exponential_',\n",
       " 'fft',\n",
       " 'fill_',\n",
       " 'flatten',\n",
       " 'flip',\n",
       " 'float',\n",
       " 'floor',\n",
       " 'floor_',\n",
       " 'fmod',\n",
       " 'fmod_',\n",
       " 'frac',\n",
       " 'frac_',\n",
       " 'gather',\n",
       " 'ge',\n",
       " 'ge_',\n",
       " 'gels',\n",
       " 'geometric_',\n",
       " 'geqrf',\n",
       " 'ger',\n",
       " 'gesv',\n",
       " 'get_device',\n",
       " 'grad',\n",
       " 'grad_fn',\n",
       " 'gt',\n",
       " 'gt_',\n",
       " 'half',\n",
       " 'hardshrink',\n",
       " 'histc',\n",
       " 'ifft',\n",
       " 'index_add',\n",
       " 'index_add_',\n",
       " 'index_copy',\n",
       " 'index_copy_',\n",
       " 'index_fill',\n",
       " 'index_fill_',\n",
       " 'index_put',\n",
       " 'index_put_',\n",
       " 'index_select',\n",
       " 'indices',\n",
       " 'int',\n",
       " 'int_repr',\n",
       " 'inverse',\n",
       " 'irfft',\n",
       " 'is_coalesced',\n",
       " 'is_complex',\n",
       " 'is_contiguous',\n",
       " 'is_cuda',\n",
       " 'is_distributed',\n",
       " 'is_floating_point',\n",
       " 'is_leaf',\n",
       " 'is_nonzero',\n",
       " 'is_pinned',\n",
       " 'is_quantized',\n",
       " 'is_same_size',\n",
       " 'is_set_to',\n",
       " 'is_shared',\n",
       " 'is_signed',\n",
       " 'is_sparse',\n",
       " 'isclose',\n",
       " 'item',\n",
       " 'kthvalue',\n",
       " 'layout',\n",
       " 'le',\n",
       " 'le_',\n",
       " 'lerp',\n",
       " 'lerp_',\n",
       " 'lgamma',\n",
       " 'lgamma_',\n",
       " 'log',\n",
       " 'log10',\n",
       " 'log10_',\n",
       " 'log1p',\n",
       " 'log1p_',\n",
       " 'log2',\n",
       " 'log2_',\n",
       " 'log_',\n",
       " 'log_normal_',\n",
       " 'log_softmax',\n",
       " 'logdet',\n",
       " 'logsumexp',\n",
       " 'long',\n",
       " 'lt',\n",
       " 'lt_',\n",
       " 'lu',\n",
       " 'lu_solve',\n",
       " 'map2_',\n",
       " 'map_',\n",
       " 'masked_fill',\n",
       " 'masked_fill_',\n",
       " 'masked_scatter',\n",
       " 'masked_scatter_',\n",
       " 'masked_select',\n",
       " 'matmul',\n",
       " 'matrix_power',\n",
       " 'max',\n",
       " 'mean',\n",
       " 'median',\n",
       " 'min',\n",
       " 'mm',\n",
       " 'mode',\n",
       " 'mul',\n",
       " 'mul_',\n",
       " 'multinomial',\n",
       " 'mv',\n",
       " 'mvlgamma',\n",
       " 'mvlgamma_',\n",
       " 'name',\n",
       " 'narrow',\n",
       " 'narrow_copy',\n",
       " 'ndimension',\n",
       " 'ne',\n",
       " 'ne_',\n",
       " 'neg',\n",
       " 'neg_',\n",
       " 'nelement',\n",
       " 'new',\n",
       " 'new_empty',\n",
       " 'new_full',\n",
       " 'new_ones',\n",
       " 'new_tensor',\n",
       " 'new_zeros',\n",
       " 'nonzero',\n",
       " 'norm',\n",
       " 'normal_',\n",
       " 'numel',\n",
       " 'numpy',\n",
       " 'orgqr',\n",
       " 'ormqr',\n",
       " 'output_nr',\n",
       " 'permute',\n",
       " 'pin_memory',\n",
       " 'pinverse',\n",
       " 'polygamma',\n",
       " 'polygamma_',\n",
       " 'potrf',\n",
       " 'potri',\n",
       " 'potrs',\n",
       " 'pow',\n",
       " 'pow_',\n",
       " 'prelu',\n",
       " 'prod',\n",
       " 'pstrf',\n",
       " 'put_',\n",
       " 'q_scale',\n",
       " 'q_zero_point',\n",
       " 'qr',\n",
       " 'quantize_linear',\n",
       " 'random_',\n",
       " 'reciprocal',\n",
       " 'reciprocal_',\n",
       " 'record_stream',\n",
       " 'register_hook',\n",
       " 'reinforce',\n",
       " 'relu',\n",
       " 'relu_',\n",
       " 'remainder',\n",
       " 'remainder_',\n",
       " 'renorm',\n",
       " 'renorm_',\n",
       " 'repeat',\n",
       " 'repeat_interleave',\n",
       " 'requires_grad',\n",
       " 'requires_grad_',\n",
       " 'reshape',\n",
       " 'reshape_as',\n",
       " 'resize',\n",
       " 'resize_',\n",
       " 'resize_as',\n",
       " 'resize_as_',\n",
       " 'retain_grad',\n",
       " 'rfft',\n",
       " 'roll',\n",
       " 'rot90',\n",
       " 'round',\n",
       " 'round_',\n",
       " 'rsqrt',\n",
       " 'rsqrt_',\n",
       " 'scatter',\n",
       " 'scatter_',\n",
       " 'scatter_add',\n",
       " 'scatter_add_',\n",
       " 'select',\n",
       " 'set_',\n",
       " 'shape',\n",
       " 'share_memory_',\n",
       " 'short',\n",
       " 'sigmoid',\n",
       " 'sigmoid_',\n",
       " 'sign',\n",
       " 'sign_',\n",
       " 'sin',\n",
       " 'sin_',\n",
       " 'sinh',\n",
       " 'sinh_',\n",
       " 'size',\n",
       " 'slogdet',\n",
       " 'smm',\n",
       " 'softmax',\n",
       " 'solve',\n",
       " 'sort',\n",
       " 'sparse_dim',\n",
       " 'sparse_mask',\n",
       " 'sparse_resize_',\n",
       " 'sparse_resize_and_clear_',\n",
       " 'split',\n",
       " 'split_with_sizes',\n",
       " 'sqrt',\n",
       " 'sqrt_',\n",
       " 'squeeze',\n",
       " 'squeeze_',\n",
       " 'sspaddmm',\n",
       " 'std',\n",
       " 'stft',\n",
       " 'storage',\n",
       " 'storage_offset',\n",
       " 'storage_type',\n",
       " 'stride',\n",
       " 'sub',\n",
       " 'sub_',\n",
       " 'sum',\n",
       " 'sum_to_size',\n",
       " 'svd',\n",
       " 'symeig',\n",
       " 't',\n",
       " 't_',\n",
       " 'take',\n",
       " 'tan',\n",
       " 'tan_',\n",
       " 'tanh',\n",
       " 'tanh_',\n",
       " 'to',\n",
       " 'to_dense',\n",
       " 'to_mkldnn',\n",
       " 'to_sparse',\n",
       " 'tolist',\n",
       " 'topk',\n",
       " 'trace',\n",
       " 'transpose',\n",
       " 'transpose_',\n",
       " 'triangular_solve',\n",
       " 'tril',\n",
       " 'tril_',\n",
       " 'triu',\n",
       " 'triu_',\n",
       " 'trtrs',\n",
       " 'trunc',\n",
       " 'trunc_',\n",
       " 'type',\n",
       " 'type_as',\n",
       " 'unbind',\n",
       " 'unfold',\n",
       " 'uniform_',\n",
       " 'unique',\n",
       " 'unique_consecutive',\n",
       " 'unsqueeze',\n",
       " 'unsqueeze_',\n",
       " 'values',\n",
       " 'var',\n",
       " 'view',\n",
       " 'view_as',\n",
       " 'where',\n",
       " 'zero_']"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[o for o  in dir(A) if not o.startswith(\"_\")]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# But what about the GPU?\n",
    "How do I use the GPU?\n",
    "\n",
    "If you have a GPU make sure that the right pytorch is installed\n",
    "(check https://pytorch.org/ for details)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cpu')"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you have a GPU you should get something like: \n",
    "`device(type='cuda', index=0)`\n",
    "\n",
    "You can move data to the GPU by doing `.to(device)`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 0., 0.],\n",
       "        [0., 1., 0.],\n",
       "        [0., 0., 1.]])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = torch.eye(3)\n",
    "data.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now the computation happens on the GPU."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[2., 0., 0.],\n",
       "        [0., 2., 0.],\n",
       "        [0., 0., 2.]])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = data + data\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cpu')"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res.device"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: before `v0.4` one had to use `.cuda()` and `.cpu()` to move stuff to and from the GPU.\n",
    "This littered the code with many:\n",
    "```python\n",
    "if CUDA:\n",
    "    model = model.cuda()\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Automatic differentiation with `autograd`\n",
    "Prior to `v0.4` PyTorch used the class `Variable` to record gradients. You had to wrap `Tensor`s in `Variable`s.\n",
    "`Variable`s behaved exactly like `Tensors`.\n",
    "\n",
    "With `v0.4` `Tensor` can record gradients directly if you tell it do do so, e.g. `torch.ones(3, requires_grad=True)`.\n",
    "There is no need for `Variable` anymore.\n",
    "Many tutorials still use `Variable`, be aware!\n",
    "\n",
    "Ref:\n",
    "- https://pytorch.org/docs/stable/autograd.html\n",
    "- https://pytorch.org/tutorials/beginner/blitz/autograd_tutorial.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You rarely use `torch.autograd` directly.\n",
    "Pretty much everything is part or `torch.Tensor` now.\n",
    "Simply add `requires_grad=True` to the tensors you want to calculate the gradients for.\n",
    "`nn.Module` track gradients automatically."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch import autograd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(2.)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.tensor(2.)\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(2., requires_grad=True)"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.tensor(2., requires_grad=True)\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "print(x.requires_grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n"
     ]
    }
   ],
   "source": [
    "print(x.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Grad of x: None\n"
     ]
    }
   ],
   "source": [
    "y = x ** 2\n",
    "\n",
    "print(\"Grad of x:\", x.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Grad of x: tensor(4.)\n"
     ]
    }
   ],
   "source": [
    "y = x ** 2\n",
    "y.backward()\n",
    "\n",
    "print(\"Grad of x:\", x.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "element 0 of tensors does not require grad and does not have a grad_fn",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-69-7af5c5d62813>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# What is going to happen here?\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2.\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/usr/local/lib/python3.7/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m    105\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    106\u001b[0m         \"\"\"\n\u001b[0;32m--> 107\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    108\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    109\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.7/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m     91\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m     92\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: element 0 of tensors does not require grad and does not have a grad_fn"
     ]
    }
   ],
   "source": [
    "# What is going to happen here?\n",
    "x = torch.tensor(2.)\n",
    "x.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n"
     ]
    }
   ],
   "source": [
    "# Don't record the gradient\n",
    "# Useful for inference\n",
    "\n",
    "params = torch.tensor(2., requires_grad=True)\n",
    "\n",
    "with torch.no_grad():\n",
    "    y = x * x\n",
    "    print(x.grad_fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`nn.Module` and `nn.Parameter` keep track of gradients for you."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[ 0.6877, -0.4699]], requires_grad=True)"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lin = nn.Linear(2, 1, bias=True)\n",
    "lin.weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.nn.parameter.Parameter"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(lin.weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "isinstance(lin.weight, torch.FloatTensor)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercise\n",
    "- Do you remember the analytical solution to solve for the parameters of linear regression? Implement it."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}