lenet5.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import csv\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data\\train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data\\train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data\\t10k-labels-idx1-ubyte.gz\n",
      "\n",
      "Image Shape: (28, 28, 1)\n",
      "\n",
      "Training Set:   55000 samples\n",
      "Validation Set: 5000 samples\n",
      "Test Set:       10000 samples\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "#mnist = input_data.read_data_sets('MNIST_data', one_hot=True)\n",
    "mnist = input_data.read_data_sets(\"MNIST_data\", reshape=False)\n",
    "X_train, y_train           = mnist.train.images, mnist.train.labels\n",
    "X_validation, y_validation = mnist.validation.images, mnist.validation.labels\n",
    "X_test, y_test             = mnist.test.images, mnist.test.labels\n",
    "\n",
    "assert(len(X_train) == len(y_train))\n",
    "assert(len(X_validation) == len(y_validation))\n",
    "assert(len(X_test) == len(y_test))\n",
    "\n",
    "print()\n",
    "print(\"Image Shape: {}\".format(X_train[0].shape))\n",
    "print()\n",
    "print(\"Training Set:   {} samples\".format(len(X_train)))\n",
    "print(\"Validation Set: {} samples\".format(len(X_validation)))\n",
    "print(\"Test Set:       {} samples\".format(len(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000,)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5000,)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_validation.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_testNew=np.concatenate((X_test,X_validation))\n",
    "y_testNew=np.concatenate((y_test,y_validation))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updated Image Shape: (32, 32, 1)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "# Pad images with 0s\n",
    "X_train      = np.pad(X_train, ((0,0),(2,2),(2,2),(0,0)), 'constant')\n",
    "X_validation = np.pad(X_validation, ((0,0),(2,2),(2,2),(0,0)), 'constant')\n",
    "X_test       = np.pad(X_test, ((0,0),(2,2),(2,2),(0,0)), 'constant')\n",
    "X_testNew    = np.pad(X_testNew, ((0,0),(2,2),(2,2),(0,0)), 'constant')\n",
    "print(\"Updated Image Shape: {}\".format(X_train[0].shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "image = X_train[1].squeeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADzBJREFUeJzt3V2sXWWdx/HvX8DMBEgGejrY8NJKQiBgDJJCTKY2EiMC\nMaGdCyIo4UI4hjhG0EmmgkFmkMaZiAo36HEg4kRUQunABc4EiaF402mhvBRO8S0gkNoXcCKECwf6\nn4u9mjnt7Ged3X32Xrunz/eTNN3nefba68/i/LrXXs9ezxOZiaT6vGfSBUiaDMMvVcrwS5Uy/FKl\nDL9UKcMvVcrwS5Uy/FKlDL9UqaMXsnFEXAzcARwF/GtmfmOe5/t1QmnMMjMGeV4M+/XeiDgK+BXw\nceBVYAtwRWa+0LKN4ZfGbNDwL+S0/wLgN5n5u8z8M/AT4LIFvJ6kDi0k/CcDr8z5+dWmTdIisKDP\n/IOIiGlgetz7kXRoFhL+14BT5/x8StN2gMycAWbAz/zS4WQhp/1bgDMi4v0R8V7gU8DDoylL0rgN\n/c6fme9ExN8B/0lvqO+ezHx+ZJVJGquhh/qG2pmn/dLYdTHUJ2kRM/xSpQy/VCnDL1XK8EuVMvxS\npQy/VCnDL1XK8EuVMvxSpQy/VCnDL1XK8EuVMvxSpQy/VCnDL1XK8EuVMvxSpQy/VCnDL1XK8EuV\nMvxSpQy/VCnDL1XK8EuVWtAqvRHxEvAm8C7wTmauHEVRksZvFEt0X5iZe0fwOpI65Gm/VKmFhj+B\nn0fEkxExPYqCJHVjoaf9qzLztYj4a+DRiNiRmZvmPqH5R8F/GKTDzMiW6I6IW4C3MvObLc9xiW5p\nzMa+RHdEHBsRx+9/DFwEbB/29SR1ayGn/ScBGyNi/+vcl5n/MZKqJI3dyE77B9qZp/3S2I39tF/S\n4mb4pUoZfqlShl+qlOGXKjWKG3u0yC1fvrzYd8011xT7brrppmJfaRSpGRrua3Z2ttj31a9+tdi3\ncePGYp/KfOeXKmX4pUoZfqlShl+qlOGXKuV3+48wS5cu7dv+la98pbjNpz/96WLfkiVLin1tV+6H\nudrf9rv4yiuvFPvOP//8Yt/evfXNMOd3+yW1MvxSpQy/VCnDL1XK8EuVMvxSpRzqW4Tabqi59dZb\n+7a3/X8ex/Dbnj17in0lU1NTxb4VK1YU+1544YVi3znnnHPIdSx2DvVJamX4pUoZfqlShl+qlOGX\nKmX4pUrNO9QXEfcAnwR2Z+YHmrYTgZ8CK4CXgMsz84/z7syhvpHYsmVLse+8887r2z7sUF/bMNqF\nF15Y7BvmbrpVq1YV+x5//PFiX9t/29FH1zdN5SiH+n4AXHxQ2zrgscw8A3is+VnSIjJv+DNzE/DG\nQc2XAfc2j+8F1oy4LkljNuxn/pMyc2fz+A/0VuyVtIgs+ANRZmbbZ/mImAamF7ofSaM17Dv/rohY\nBtD8vbv0xMycycyVmblyyH1JGoNhw/8wcHXz+GrgodGUI6krgwz1/Rj4KDAF7AK+Bvw7cD9wGvAy\nvaG+gy8K9nsth/oGdNZZZxX72ob6Xn/99b7tbXfZtQ3L3XDDDcW+66+/vti3fv36vu2///3vi9u0\nafs93bdvX7Hvuuuu69s+MzMzVB2LwaBDffN+5s/MKwpdHzukiiQdVvyGn1Qpwy9VyvBLlTL8UqUM\nv1QpJ/BchNqGAUvDdsOuWTc9Xf5y5l133VXsK62f99RTTxW3Wbt2bbHvgQceKPa1/Q6/733v69t+\nJK/h5wSekloZfqlShl+qlOGXKmX4pUoZfqlS9c1ueATYsWNHZ/tquxvwxRdfLPaV7i5su0tw3bry\nVJBtk4y2DdsdyUN6C+U7v1Qpwy9VyvBLlTL8UqUMv1Qpr/YfYVavXt23ve1moLYr+rOzs8W+M888\ns9i3efPmvu1Lly4tbtN2g05bjZdcckmxT2W+80uVMvxSpQy/VCnDL1XK8EuVMvxSpeYd6ouIe4BP\nArsz8wNN2y3AtcD+8ZcbM/ORcRWpwV155ZV926+99triNm03zbQNv7VtVxrSG/YGnTvvvLPY1zYv\noMoGeef/AXBxn/ZvZ+a5zR+DLy0y84Y/MzcB8y7CKWlxWchn/i9ExLMRcU9EnDCyiiR1Ytjw3wWc\nDpwL7ARuLz0xIqYjYmtEbB1yX5LGYKjwZ+auzHw3M/cB3wcuaHnuTGauzMyVwxYpafSGCn9ELJvz\n41pg+2jKkdSVQYb6fgx8FJiKiFeBrwEfjYhzgQReAj43xho1AsMuyzbq7Z544oniNl/60peKfQ7n\njd684c/MK/o03z2GWiR1yG/4SZUy/FKlDL9UKcMvVcrwS5VyAs8jzH333de3ffny5cVtpqamin1t\nE38ee+yxgxfWuPnmm4t9Dud1y3d+qVKGX6qU4ZcqZfilShl+qVKGX6pUDHvX1lA7i+huZxqJtqG+\nr3/968W+NWvW9G3ftm1bcZu2NffaJvfUgTKzPEvqHL7zS5Uy/FKlDL9UKcMvVcrwS5Xyav8IlJam\nAtizZ0+x70j2s5/9rG/7Jz7xieI2bXP4fec731lwTbXwar+kVoZfqpThlypl+KVKGX6pUoZfqtQg\ny3WdCvwQOIne8lwzmXlHRJwI/BRYQW/Jrssz84/jK3XyVq9e3bf99tuLixSzY8eOYt9VV1214JoO\nV7fddlvf9osuuqi4zZlnnjmuctTHIO/87wBfzsyzgQ8Dn4+Is4F1wGOZeQbwWPOzpEVi3vBn5s7M\nfKp5/CYwC5wMXAbc2zztXqD/PZySDkuH9Jk/IlYAHwI2Aydl5s6m6w/0PhZIWiQGnrc/Io4DNgDX\nZ+afIv7vG4SZmaWv7kbENDC90EIljdZA7/wRcQy94P8oMx9smndFxLKmfxmwu9+2mTmTmSszc+Uo\nCpY0GvOGP3pv8XcDs5n5rTldDwNXN4+vBh4afXmSxmWQ0/6/Aa4CnouIp5u2G4FvAPdHxGeBl4HL\nx1Nit9ru0Pvud7/bt3337r4nPcCRPZzXtlzX9773vb7tcz8uarLmDX9m/hIo/R/72GjLkdQVv+En\nVcrwS5Uy/FKlDL9UKcMvVWrgb/jVYu3atcW+0l1njz/++LjKmbi25bo2bNhQ7Csdq7YJY9vugNTo\n+c4vVcrwS5Uy/FKlDL9UKcMvVcrwS5VyqO8gmzZtKva95z39/60sTewJ8JnPfKbYNzs7W+x78skn\ni31tli9f3rf9Ix/5SHGbtuHNNWvKs7O13aFXGtK74447itu09Wn0fOeXKmX4pUoZfqlShl+qlOGX\nKhVtN1qMfGeF6b0XiwceeKBv+6iviANs27Zt8MLmOO200/q2L1mypLjNsDW2bVdaruvOO+8sbrN3\n795inwaXmQNNlOg7v1Qpwy9VyvBLlTL8UqUMv1Qpwy9Vat6hvog4FfghvSW4E5jJzDsi4hbgWmBP\n89QbM/OReV5rUQ/1lZbyeuSR8n/2ypXl9Un37dtX7Bv18FvbNm+//Xaxr21evfXr1xf7Nm7cWOzT\neA061DfIXX3vAF/OzKci4njgyYh4tOn7dmZ+c9giJU3OIGv17QR2No/fjIhZ4ORxFyZpvA7pM39E\nrAA+BGxumr4QEc9GxD0RccKIa5M0RgOHPyKOAzYA12fmn4C7gNOBc+mdGdxe2G46IrZGxNYR1Ctp\nRAYKf0QcQy/4P8rMBwEyc1dmvpuZ+4DvAxf02zYzZzJzZWaWr3xJ6ty84Y/e5eO7gdnM/Nac9mVz\nnrYW2D768iSNyyBDfauAJ4DngP1jUzcCV9A75U/gJeBzzcXBttda1EN9JVNTU8W+W2+9dajXnJ6e\nLvY9+OCDxb5h7oxrmzvPJbQWn5EN9WXmL4F+L9Y6pi/p8OY3/KRKGX6pUoZfqpThlypl+KVKOYGn\ndIRxAk9JrQy/VCnDL1XK8EuVMvxSpQy/VCnDL1XK8EuVMvxSpQy/VCnDL1XK8EuVMvxSpQy/VCnD\nL1XK8EuVMvxSpQy/VCnDL1VqkLX6/iIi/isinomI5yPiH5v2EyPi0Yj4dfO3S3RLi8gga/UFcGxm\nvtWs1vtL4IvA3wJvZOY3ImIdcEJm/sM8r+UEntKYjWwCz+x5q/nxmOZPApcB9zbt9wJrhqhT0oQM\n9Jk/Io6KiKeB3cCjmbkZOGnOqrx/AE4aU42SxmCg8Gfmu5l5LnAKcEFEfOCg/qR3NvD/RMR0RGyN\niK0LrlbSyBzS1f7M/G/gF8DFwK6IWAbQ/L27sM1MZq7MzJULLVbS6AxytX9pRPxV8/gvgY8DO4CH\ngaubp10NPDSuIiWN3iBX+z9I74LeUfT+sbg/M/8pIpYA9wOnAS8Dl2fmG/O8llf7pTEb9Gq/a/VJ\nRxjX6pPUyvBLlTL8UqUMv1Qpwy9V6uiO97eX3rAgwFTz86RZx4Gs40CLrY7lg75gp0N9B+w4Yuvh\n8K0/67COWuvwtF+qlOGXKjXJ8M9McN9zWceBrONAR2wdE/vML2myPO2XKjWR8EfExRHxYkT8ppn/\nbyIi4qWIeC4inu5yspGIuCcidkfE9jltnU+IWqjjloh4rTkmT0fEpR3UcWpE/CIiXmgmif1i097p\nMWmpo9Nj0tmkuZnZ6R96twb/FjgdeC/wDHB213U0tbwETE1gv6uB84Dtc9r+BVjXPF4H/POE6rgF\n+PuOj8cy4Lzm8fHAr4Czuz4mLXV0ekyAAI5rHh8DbAY+POrjMYl3/guA32Tm7zLzz8BP6E0GWo3M\n3AQcPPdB5xOiFuroXGbuzMynmsdvArPAyXR8TFrq6FT2jH3S3EmE/2TglTk/v8oEDnAjgZ9HxJMR\nMT2hGvY7nCZE/UJEPNt8LOh0PYaIWAF8iN673cSOyUF1QMfHpItJc2u/4LcqexOTXgJ8PiJWT7og\naJ8QtQN30ftIdi6wE7i9qx1HxHHABuD6zPzT3L4uj0mfOjo/JrmASXMHNYnwvwacOufnU5q2zmXm\na83fu4GN9D6STMpAE6KOW2buan7x9gHfp6Nj0iwIswH4UWY+2DR3fkz61TGpY9Ls+5AnzR3UJMK/\nBTgjIt4fEe8FPkVvMtBORcSxEXH8/sfARcD29q3G6rCYEHX/L1djLR0ck2ZVqLuB2cz81pyuTo9J\nqY6uj0lnk+Z2dQXzoKuZl9K7kvpb4KYJ1XA6vZGGZ4Dnu6wD+DG908f/oXfN47PAEuAx4NfAz4ET\nJ1THvwHPAc82v2zLOqhjFb1T2GeBp5s/l3Z9TFrq6PSYAB8EtjX72w7c3LSP9Hj4DT+pUrVf8JOq\nZfilShl+qVKGX6qU4ZcqZfilShl+qVKGX6rU/wKbrqVJR1IMowAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1b18001c208>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(image, cmap=\"gray\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.utils import shuffle\n",
    "X_train, y_train = shuffle(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "EPOCHS = 10\n",
    "BATCH_SIZE = 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from tensorflow.contrib.layers import flatten\n",
    "\n",
    "def LeNet(x):    \n",
    "    # Hyperparameters\n",
    "    mu = 0\n",
    "    sigma = 0.1\n",
    "    \n",
    "    # SOLUTION: Layer 1: Convolutional. Input = 32x32x1. Output = 28x28x6.\n",
    "    conv1_W = tf.Variable(tf.truncated_normal(shape=(5, 5, 1, 16), mean = mu, stddev = sigma),name='conv1_W')\n",
    "    conv1_b = tf.Variable(tf.zeros(16),name='conv1_b')\n",
    "    conv1   = tf.nn.conv2d(x, conv1_W, strides=[1, 1, 1, 1], padding='VALID') + conv1_b\n",
    "\n",
    "    # SOLUTION: Activation.\n",
    "    conv1 = tf.nn.relu(conv1)\n",
    "\n",
    "    # SOLUTION: Pooling. Input = 28x28x6. Output = 14x14x6.\n",
    "    conv1 = tf.nn.max_pool(conv1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='VALID')\n",
    "\n",
    "    # SOLUTION: Layer 2: Convolutional. Output = 10x10x16.\n",
    "    conv2_W = tf.Variable(tf.truncated_normal(shape=(5, 5, 16, 32), mean = mu, stddev = sigma),name='conv2_W')\n",
    "    conv2_b = tf.Variable(tf.zeros(32),name='conv2_b')\n",
    "    conv2   = tf.nn.conv2d(conv1, conv2_W, strides=[1, 1, 1, 1], padding='VALID') + conv2_b\n",
    "    \n",
    "    # SOLUTION: Activation.\n",
    "    conv2 = tf.nn.relu(conv2)\n",
    "\n",
    "    # SOLUTION: Pooling. Input = 10x10x16. Output = 5x5x16.\n",
    "    conv2 = tf.nn.max_pool(conv2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='VALID')\n",
    "\n",
    "    # SOLUTION: Flatten. Input = 5x5x16. Output = 400.\n",
    "    fc0   = flatten(conv2)\n",
    "    \n",
    "    # SOLUTION: Layer 3: Fully Connected. Input = 400. Output = 120.\n",
    "    fc1_W = tf.Variable(tf.truncated_normal(shape=(400, 120), mean = mu, stddev = sigma),name='fc1_W')\n",
    "    fc1_b = tf.Variable(tf.zeros(120),name='fc1_b')\n",
    "    fc1   = tf.matmul(fc0, fc1_W) + fc1_b\n",
    "    \n",
    "    # SOLUTION: Activation.\n",
    "    fc1    = tf.nn.relu(fc1)\n",
    "\n",
    "    # SOLUTION: Layer 4: Fully Connected. Input = 120. Output = 84.\n",
    "    fc2_W  = tf.Variable(tf.truncated_normal(shape=(120, 84), mean = mu, stddev = sigma),name='fc2_W')\n",
    "    fc2_b  = tf.Variable(tf.zeros(84),name='fc2_b')\n",
    "    fc2    = tf.matmul(fc1, fc2_W) + fc2_b\n",
    "    \n",
    "    # SOLUTION: Activation.\n",
    "    fc2    = tf.nn.relu(fc2)\n",
    "\n",
    "    # SOLUTION: Layer 5: Fully Connected. Input = 84. Output = 10.\n",
    "    fc3_W  = tf.Variable(tf.truncated_normal(shape=(84, 10), mean = mu, stddev = sigma),name='fc3_W')\n",
    "    fc3_b  = tf.Variable(tf.zeros(10),name='fc3_b')\n",
    "    logits = tf.matmul(fc2, fc3_W) + fc3_b\n",
    "    #res = Categorical(logits)\n",
    "    return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = tf.placeholder(tf.float32, (None, 32, 32, 1))\n",
    "y = tf.placeholder(tf.int32, (None))\n",
    "one_hot_y = tf.one_hot(y, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Dimensions must be equal, but are 800 and 400 for 'MatMul' (op: 'MatMul') with input shapes: [?,800], [400,120].",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[1;32mC:\\Users\\syale\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\common_shapes.py\u001b[0m in \u001b[0;36m_call_cpp_shape_fn_impl\u001b[1;34m(op, input_tensors_needed, input_tensors_as_shapes_needed, require_shape_fn)\u001b[0m\n\u001b[0;32m    685\u001b[0m           \u001b[0mgraph_def_version\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnode_def_str\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput_shapes\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput_tensors\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 686\u001b[1;33m           input_tensors_as_shapes, status)\n\u001b[0m\u001b[0;32m    687\u001b[0m   \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mInvalidArgumentError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\syale\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\errors_impl.py\u001b[0m in \u001b[0;36m__exit__\u001b[1;34m(self, type_arg, value_arg, traceback_arg)\u001b[0m\n\u001b[0;32m    472\u001b[0m             \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mc_api\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_Message\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstatus\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstatus\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 473\u001b[1;33m             c_api.TF_GetCode(self.status.status))\n\u001b[0m\u001b[0;32m    474\u001b[0m     \u001b[1;31m# Delete the underlying status object from memory otherwise it stays alive\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mInvalidArgumentError\u001b[0m: Dimensions must be equal, but are 800 and 400 for 'MatMul' (op: 'MatMul') with input shapes: [?,800], [400,120].",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-18-6e4a93d6a6ae>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[0mrate\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0.001\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[0mlogits\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mLeNet\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      5\u001b[0m \u001b[0mcross_entropy\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msoftmax_cross_entropy_with_logits\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlogits\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlogits\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mone_hot_y\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[0mloss_operation\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreduce_mean\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcross_entropy\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-16-7806295a23fe>\u001b[0m in \u001b[0;36mLeNet\u001b[1;34m(x)\u001b[0m\n\u001b[0;32m     34\u001b[0m     \u001b[0mfc1_W\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mVariable\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtruncated_normal\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m400\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m120\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmean\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmu\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstddev\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msigma\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'fc1_W'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     35\u001b[0m     \u001b[0mfc1_b\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mVariable\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m120\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'fc1_b'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 36\u001b[1;33m     \u001b[0mfc1\u001b[0m   \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfc0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfc1_W\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mfc1_b\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     37\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     38\u001b[0m     \u001b[1;31m# SOLUTION: Activation.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\syale\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\math_ops.py\u001b[0m in \u001b[0;36mmatmul\u001b[1;34m(a, b, transpose_a, transpose_b, adjoint_a, adjoint_b, a_is_sparse, b_is_sparse, name)\u001b[0m\n\u001b[0;32m   1889\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1890\u001b[0m       return gen_math_ops._mat_mul(\n\u001b[1;32m-> 1891\u001b[1;33m           a, b, transpose_a=transpose_a, transpose_b=transpose_b, name=name)\n\u001b[0m\u001b[0;32m   1892\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1893\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\syale\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\gen_math_ops.py\u001b[0m in \u001b[0;36m_mat_mul\u001b[1;34m(a, b, transpose_a, transpose_b, name)\u001b[0m\n\u001b[0;32m   2434\u001b[0m     _, _, _op = _op_def_lib._apply_op_helper(\n\u001b[0;32m   2435\u001b[0m         \u001b[1;34m\"MatMul\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0ma\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mb\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtranspose_a\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtranspose_a\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtranspose_b\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtranspose_b\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2436\u001b[1;33m         name=name)\n\u001b[0m\u001b[0;32m   2437\u001b[0m     \u001b[0m_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_op\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2438\u001b[0m     \u001b[0m_inputs_flat\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_op\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\syale\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py\u001b[0m in \u001b[0;36m_apply_op_helper\u001b[1;34m(self, op_type_name, name, **keywords)\u001b[0m\n\u001b[0;32m    785\u001b[0m         op = g.create_op(op_type_name, inputs, output_types, name=scope,\n\u001b[0;32m    786\u001b[0m                          \u001b[0minput_types\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minput_types\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mattrs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mattr_protos\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 787\u001b[1;33m                          op_def=op_def)\n\u001b[0m\u001b[0;32m    788\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0moutput_structure\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mop_def\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_stateful\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mop\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    789\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\syale\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\u001b[0m in \u001b[0;36mcreate_op\u001b[1;34m(self, op_type, inputs, dtypes, input_types, name, attrs, op_def, compute_shapes, compute_device)\u001b[0m\n\u001b[0;32m   2956\u001b[0m         op_def=op_def)\n\u001b[0;32m   2957\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mcompute_shapes\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2958\u001b[1;33m       \u001b[0mset_shapes_for_outputs\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mret\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2959\u001b[0m     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_add_op\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mret\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2960\u001b[0m     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_record_op_seen_by_control_dependencies\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mret\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\syale\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\u001b[0m in \u001b[0;36mset_shapes_for_outputs\u001b[1;34m(op)\u001b[0m\n\u001b[0;32m   2207\u001b[0m       \u001b[0mshape_func\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_call_cpp_shape_fn_and_require_op\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2208\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2209\u001b[1;33m   \u001b[0mshapes\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mshape_func\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mop\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2210\u001b[0m   \u001b[1;32mif\u001b[0m \u001b[0mshapes\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2211\u001b[0m     raise RuntimeError(\n",
      "\u001b[1;32mC:\\Users\\syale\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\u001b[0m in \u001b[0;36mcall_with_requiring\u001b[1;34m(op)\u001b[0m\n\u001b[0;32m   2157\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2158\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0mcall_with_requiring\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mop\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2159\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mcall_cpp_shape_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mop\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrequire_shape_fn\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2160\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2161\u001b[0m   \u001b[0m_call_cpp_shape_fn_and_require_op\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcall_with_requiring\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\syale\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\common_shapes.py\u001b[0m in \u001b[0;36mcall_cpp_shape_fn\u001b[1;34m(op, require_shape_fn)\u001b[0m\n\u001b[0;32m    625\u001b[0m     res = _call_cpp_shape_fn_impl(op, input_tensors_needed,\n\u001b[0;32m    626\u001b[0m                                   \u001b[0minput_tensors_as_shapes_needed\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 627\u001b[1;33m                                   require_shape_fn)\n\u001b[0m\u001b[0;32m    628\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mres\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdict\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    629\u001b[0m       \u001b[1;31m# Handles the case where _call_cpp_shape_fn_impl calls unknown_shape(op).\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\syale\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\common_shapes.py\u001b[0m in \u001b[0;36m_call_cpp_shape_fn_impl\u001b[1;34m(op, input_tensors_needed, input_tensors_as_shapes_needed, require_shape_fn)\u001b[0m\n\u001b[0;32m    689\u001b[0m       \u001b[0mmissing_shape_fn\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    690\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 691\u001b[1;33m       \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0merr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    692\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    693\u001b[0m   \u001b[1;32mif\u001b[0m \u001b[0mmissing_shape_fn\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Dimensions must be equal, but are 800 and 400 for 'MatMul' (op: 'MatMul') with input shapes: [?,800], [400,120]."
     ]
    }
   ],
   "source": [
    "\n",
    "rate = 0.001\n",
    "\n",
    "logits = LeNet(x)\n",
    "cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=one_hot_y)\n",
    "loss_operation = tf.reduce_mean(cross_entropy)\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate = rate)\n",
    "training_operation = optimizer.minimize(loss_operation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(one_hot_y, 1))\n",
    "accuracy_operation = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "saver = tf.train.Saver()\n",
    "\n",
    "def evaluate(X_data, y_data):\n",
    "    num_examples = len(X_data)\n",
    "    total_accuracy = 0\n",
    "    sess = tf.get_default_session()\n",
    "    #for offset in range(0, num_examples, BATCH_SIZE):\n",
    "    #    batch_x, batch_y = X_data[offset:offset+BATCH_SIZE], y_data[offset:offset+BATCH_SIZE]\n",
    "    accuracy = sess.run(accuracy_operation, feed_dict={x: X_data, y:y_data})\n",
    "    #    total_accuracy += (accuracy * len(batch_x))\n",
    "    #return total_accuracy / num_examples\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training...\n",
      "\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'training_operation' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-15-582b314d38ff>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     15\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m                 \u001b[0mbatch_x\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_y\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mX_train\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0moffset\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mend\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0moffset\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mend\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 17\u001b[1;33m                 \u001b[0msess\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtraining_operation\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m{\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mbatch_x\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mbatch_y\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     18\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     19\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mj\u001b[0m\u001b[1;33m%\u001b[0m\u001b[1;36m500\u001b[0m\u001b[1;33m==\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'training_operation' is not defined"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    num_examples = len(X_train)\n",
    "    \n",
    "    print(\"Training...\")\n",
    "    print()\n",
    "    j=0\n",
    "    with open('lenet_1632.csv', 'a') as f:\n",
    "        for i in range(5):\n",
    "            X_train, y_train = shuffle(X_train, y_train)\n",
    "            batch_x=tf.placeholder(tf.float32,shape=(128,32,32,1))\n",
    "            for offset in range(0, num_examples, BATCH_SIZE):\n",
    "                j+=1\n",
    "                end = offset + BATCH_SIZE\n",
    "\n",
    "                batch_x, batch_y = X_train[offset:end], y_train[offset:end]\n",
    "                sess.run(training_operation, feed_dict={x: batch_x, y: batch_y})\n",
    "                \n",
    "                if j%500==0:\n",
    "                    training_accuracy=evaluate(X_train,y_train)\n",
    "                    validation_accuracy = evaluate(X_validation, y_validation)\n",
    "                    test_accuracy = evaluate(X_test, y_test)\n",
    "                    whole_accuracy=evaluate(X_testNew,y_testNew)\n",
    "                    print(\"EPOCH {} ...\".format(j))\n",
    "                    print(\"Validation Accuracy = {:.3f}\".format(validation_accuracy))\n",
    "                    print(\"Test Accuracy = {:.3f}\".format(test_accuracy))\n",
    "                    print(\"whole test Accuracy={:.3f}\".format(whole_accuracy))\n",
    "                    print(\"training Accuracy=={:.3f}\".format(training_accuracy))\n",
    "                    writer = csv.writer(f, delimiter=',')\n",
    "                    writer.writerow([training_accuracy,validation_accuracy,test_accuracy,whole_accuracy])\n",
    "                \n",
    "\n",
    "    saver.save(sess, './lenet')\n",
    "    print(\"Model saved\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from .\\lenet.ckpt\n",
      "Test Accuracy = 0.965\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, tf.train.latest_checkpoint('.'))\n",
    "\n",
    "    test_accuracy = evaluate(X_test, y_test)\n",
    "    print(\"Test Accuracy = {:.3f}\".format(test_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}