6_CNN/5_CIFAR10/2_Batch_Normalization_79.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "2_Batch_Normalization_79.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.8"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OaqojFwVWlZG"
      },
      "source": [
        "# Implementing Batch Normalization on CIFAR10\n",
        "\n",
        "## Batch Normalization\n",
        "(See [5_Neural_Network/2_Tensorflow.ipynb](../../5_Neural_Network/2_Tensorflow.ipynb))\n",
        "\n",
        "* Distribution on each mini-batch keeps changing.\n",
        "* In order to learn changed distribuition, model deviates a little from correct direction.\n",
        "* **Batch Normalization** prevents the mean of weights on each batch from being biased toward + or -, so that backpropagate weights smoothly.\n",
        "* **Batch Normalization** is the concept of normalizing the output of the middle layer so that the next layer learns normalized output from the previous layer.\n",
        "\n",
        "$$BN(x_i) = \\gamma \\cdot \\frac{x_i - \\mu_B}{\\sqrt{{\\sigma_B}^2 + \\epsilon}}$$\n",
        "\n",
        "in keras: ```model.add(tf.keras.layers.BatchNormalization())```\n",
        "\n",
        "[Batch Normalization: Accelerating Deep Network Training by Reducing Internal Coveriance Shift](https://arxiv.org/abs/1502.03167)\n",
        "\n",
        "This improves accuracy on CIFAR10 learning (74% to 79%)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VCkOge6SnTKx",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8bfb6b45-2742-4746-cfe2-ec40bdce0341"
      },
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow.keras import layers, models\n",
        "from tensorflow.keras import datasets \n",
        "from tensorflow.keras.utils import to_categorical\n",
        "\n",
        "# load CIFAR10 Dataset\n",
        "(x_train, y_train), (x_test, y_test) = datasets.cifar10.load_data()\n",
        "Y_train = to_categorical(y_train)\n",
        "Y_test = to_categorical(y_test)\n",
        "\n",
        "print(\"Length of train set:\", len(Y_train))\n",
        "print(\"Shape of x_train:\", x_train.shape[1:])"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz\n",
            "170500096/170498071 [==============================] - 4s 0us/step\n",
            "Length of train set: 50000\n",
            "Shape of x_train: (32, 32, 3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SU1ARsYfn5kD",
        "outputId": "bf416c20-5b8a-4863-907e-0c9baf870d25"
      },
      "source": [
        "img_rows, img_cols, channel = x_train.shape[1:]\n",
        "\n",
        "# Unifying image size (reshape X_train, X_test)\n",
        "X_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, channel)\n",
        "X_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, channel)\n",
        "input_shape = (img_rows, img_cols, channel)\n",
        "\n",
        "# Normalize pixel value in image\n",
        "X_train = X_train.astype('float32')\n",
        "X_test = X_test.astype('float32')\n",
        "X_train /= 255\n",
        "X_test /= 255\n",
        "\n",
        "# Label is already one-hot encoded\n",
        "print(Y_train[0])\n",
        "num_classes = 10\n",
        "batch_size = 32\n",
        "print(input_shape)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n",
            "(32, 32, 3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EfN0bXAkoFYn",
        "outputId": "7b8d8154-572d-4cbd-d348-f01c8fe3438d"
      },
      "source": [
        "x = layers.Input(shape=input_shape,  name='input')\n",
        "h = layers.BatchNormalization()(x) # Batch Normalization\n",
        "h = layers.Conv2D(32, kernel_size=(3, 3), activation='relu',  name='conv1')(h)\n",
        "h = layers.Dropout(0.2)(h)\n",
        "h = layers.BatchNormalization()(h) # Batch Normalization\n",
        "h = layers.Conv2D(32, kernel_size=(3, 3), activation='relu', padding='same', name='conv2')(h)\n",
        "h = layers.MaxPooling2D(pool_size=(2, 2), name='pool1')(h)\n",
        "h = layers.BatchNormalization()(h) # Batch Normalization\n",
        "h = layers.Conv2D(64, kernel_size=(3, 3), activation='relu', padding='same', name='conv3')(h)\n",
        "h = layers.MaxPooling2D(pool_size=(2, 2), name='pool2')(h)\n",
        "h = layers.Flatten()(h)\n",
        "h = layers.Dropout(0.2)(h)\n",
        "h = layers.BatchNormalization()(h) # Batch Normalization\n",
        "h = layers.Dense(512, activation='relu', name='hidden')(h)\n",
        "h = layers.Dropout(0.2)(h)\n",
        "h = layers.BatchNormalization()(h) # Batch Normalization\n",
        "y = layers.Dense(num_classes, activation='softmax', name='output')(h)\n",
        "\n",
        "model = models.Model(x, y)\n",
        "print(model.summary())"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input (InputLayer)           [(None, 32, 32, 3)]       0         \n",
            "_________________________________________________________________\n",
            "batch_normalization (BatchNo (None, 32, 32, 3)         12        \n",
            "_________________________________________________________________\n",
            "conv1 (Conv2D)               (None, 30, 30, 32)        896       \n",
            "_________________________________________________________________\n",
            "dropout (Dropout)            (None, 30, 30, 32)        0         \n",
            "_________________________________________________________________\n",
            "batch_normalization_1 (Batch (None, 30, 30, 32)        128       \n",
            "_________________________________________________________________\n",
            "conv2 (Conv2D)               (None, 30, 30, 32)        9248      \n",
            "_________________________________________________________________\n",
            "pool1 (MaxPooling2D)         (None, 15, 15, 32)        0         \n",
            "_________________________________________________________________\n",
            "batch_normalization_2 (Batch (None, 15, 15, 32)        128       \n",
            "_________________________________________________________________\n",
            "conv3 (Conv2D)               (None, 15, 15, 64)        18496     \n",
            "_________________________________________________________________\n",
            "pool2 (MaxPooling2D)         (None, 7, 7, 64)          0         \n",
            "_________________________________________________________________\n",
            "flatten (Flatten)            (None, 3136)              0         \n",
            "_________________________________________________________________\n",
            "dropout_1 (Dropout)          (None, 3136)              0         \n",
            "_________________________________________________________________\n",
            "batch_normalization_3 (Batch (None, 3136)              12544     \n",
            "_________________________________________________________________\n",
            "hidden (Dense)               (None, 512)               1606144   \n",
            "_________________________________________________________________\n",
            "dropout_2 (Dropout)          (None, 512)               0         \n",
            "_________________________________________________________________\n",
            "batch_normalization_4 (Batch (None, 512)               2048      \n",
            "_________________________________________________________________\n",
            "output (Dense)               (None, 10)                5130      \n",
            "=================================================================\n",
            "Total params: 1,654,774\n",
            "Trainable params: 1,647,344\n",
            "Non-trainable params: 7,430\n",
            "_________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SoOPIUOJWlZZ",
        "outputId": "d43354df-fe48-42c4-f0a7-4211b1d91e0d"
      },
      "source": [
        "epochs = 25\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "history = model.fit(X_train, Y_train, batch_size=batch_size,\n",
        "          epochs=epochs, validation_split=0.1, verbose=2)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/25\n",
            "1407/1407 - 39s - loss: 1.3825 - accuracy: 0.5258 - val_loss: 0.9884 - val_accuracy: 0.6540\n",
            "Epoch 2/25\n",
            "1407/1407 - 7s - loss: 0.9660 - accuracy: 0.6599 - val_loss: 0.8193 - val_accuracy: 0.7170\n",
            "Epoch 3/25\n",
            "1407/1407 - 7s - loss: 0.8282 - accuracy: 0.7101 - val_loss: 0.7393 - val_accuracy: 0.7464\n",
            "Epoch 4/25\n",
            "1407/1407 - 7s - loss: 0.7304 - accuracy: 0.7437 - val_loss: 0.7472 - val_accuracy: 0.7486\n",
            "Epoch 5/25\n",
            "1407/1407 - 7s - loss: 0.6470 - accuracy: 0.7731 - val_loss: 0.7050 - val_accuracy: 0.7680\n",
            "Epoch 6/25\n",
            "1407/1407 - 7s - loss: 0.5865 - accuracy: 0.7929 - val_loss: 0.6452 - val_accuracy: 0.7888\n",
            "Epoch 7/25\n",
            "1407/1407 - 6s - loss: 0.5271 - accuracy: 0.8147 - val_loss: 0.6438 - val_accuracy: 0.7852\n",
            "Epoch 8/25\n",
            "1407/1407 - 7s - loss: 0.4902 - accuracy: 0.8264 - val_loss: 0.6415 - val_accuracy: 0.7892\n",
            "Epoch 9/25\n",
            "1407/1407 - 7s - loss: 0.4498 - accuracy: 0.8424 - val_loss: 0.6542 - val_accuracy: 0.7956\n",
            "Epoch 10/25\n",
            "1407/1407 - 7s - loss: 0.4174 - accuracy: 0.8515 - val_loss: 0.6628 - val_accuracy: 0.7866\n",
            "Epoch 11/25\n",
            "1407/1407 - 7s - loss: 0.3981 - accuracy: 0.8598 - val_loss: 0.6663 - val_accuracy: 0.7926\n",
            "Epoch 12/25\n",
            "1407/1407 - 6s - loss: 0.3700 - accuracy: 0.8693 - val_loss: 0.7000 - val_accuracy: 0.7878\n",
            "Epoch 13/25\n",
            "1407/1407 - 6s - loss: 0.3587 - accuracy: 0.8746 - val_loss: 0.6651 - val_accuracy: 0.7980\n",
            "Epoch 14/25\n",
            "1407/1407 - 7s - loss: 0.3476 - accuracy: 0.8763 - val_loss: 0.6759 - val_accuracy: 0.7982\n",
            "Epoch 15/25\n",
            "1407/1407 - 6s - loss: 0.3198 - accuracy: 0.8880 - val_loss: 0.6584 - val_accuracy: 0.8010\n",
            "Epoch 16/25\n",
            "1407/1407 - 6s - loss: 0.3003 - accuracy: 0.8941 - val_loss: 0.6705 - val_accuracy: 0.8008\n",
            "Epoch 17/25\n",
            "1407/1407 - 7s - loss: 0.2970 - accuracy: 0.8946 - val_loss: 0.6976 - val_accuracy: 0.7954\n",
            "Epoch 18/25\n",
            "1407/1407 - 7s - loss: 0.2822 - accuracy: 0.9006 - val_loss: 0.6926 - val_accuracy: 0.7982\n",
            "Epoch 19/25\n",
            "1407/1407 - 7s - loss: 0.2782 - accuracy: 0.9021 - val_loss: 0.7271 - val_accuracy: 0.7986\n",
            "Epoch 20/25\n",
            "1407/1407 - 7s - loss: 0.2663 - accuracy: 0.9064 - val_loss: 0.6950 - val_accuracy: 0.8026\n",
            "Epoch 21/25\n",
            "1407/1407 - 7s - loss: 0.2520 - accuracy: 0.9103 - val_loss: 0.6932 - val_accuracy: 0.8062\n",
            "Epoch 22/25\n",
            "1407/1407 - 7s - loss: 0.2463 - accuracy: 0.9132 - val_loss: 0.7280 - val_accuracy: 0.8022\n",
            "Epoch 23/25\n",
            "1407/1407 - 7s - loss: 0.2344 - accuracy: 0.9181 - val_loss: 0.7094 - val_accuracy: 0.8148\n",
            "Epoch 24/25\n",
            "1407/1407 - 6s - loss: 0.2389 - accuracy: 0.9172 - val_loss: 0.7335 - val_accuracy: 0.8134\n",
            "Epoch 25/25\n",
            "1407/1407 - 7s - loss: 0.2257 - accuracy: 0.9201 - val_loss: 0.6963 - val_accuracy: 0.8074\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jH3qljoXWlZa",
        "outputId": "76055e39-e7aa-486d-a4c7-6f44c3a34619"
      },
      "source": [
        "score = model.evaluate(X_test, Y_test)\n",
        "print()\n",
        "print('Test loss:', score[0])\n",
        "print('Test accuracy:', score[1])"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "313/313 [==============================] - 1s 3ms/step - loss: 0.7624 - accuracy: 0.7896\n",
            "\n",
            "Test loss: 0.7623740434646606\n",
            "Test accuracy: 0.7896000146865845\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}