13.1 Built LeNet and test on MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's construct LeNet in Keras!\n",
    "\n",
    "#### First let's load and prep our MNIST data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D, ZeroPadding2D\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.regularizers import l2\n",
    "from keras.datasets import mnist\n",
    "from keras.utils import np_utils\n",
    "import keras\n",
    "\n",
    "# loads the MNIST dataset\n",
    "(x_train, y_train), (x_test, y_test)  = mnist.load_data()\n",
    "\n",
    "# Lets store the number of rows and columns\n",
    "img_rows = x_train[0].shape[0]\n",
    "img_cols = x_train[1].shape[0]\n",
    "\n",
    "# Getting our date in the right 'shape' needed for Keras\n",
    "# We need to add a 4th dimenion to our date thereby changing our\n",
    "# Our original image shape of (60000,28,28) to (60000,28,28,1)\n",
    "x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)\n",
    "x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)\n",
    "\n",
    "# store the shape of a single image \n",
    "input_shape = (img_rows, img_cols, 1)\n",
    "\n",
    "# change our image type to float32 data type\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "\n",
    "# Normalize our data by changing the range from (0 to 255) to (0 to 1)\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "\n",
    "# Now we one hot encode outputs\n",
    "y_train = np_utils.to_categorical(y_train)\n",
    "y_test = np_utils.to_categorical(y_test)\n",
    "\n",
    "num_classes = y_test.shape[1]\n",
    "num_pixels = x_train.shape[1] * x_train.shape[2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now let's create our layers to replicate LeNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_3 (Conv2D)            (None, 28, 28, 20)        520       \n",
      "_________________________________________________________________\n",
      "activation_5 (Activation)    (None, 28, 28, 20)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2 (None, 14, 14, 20)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 14, 14, 50)        25050     \n",
      "_________________________________________________________________\n",
      "activation_6 (Activation)    (None, 14, 14, 50)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_4 (MaxPooling2 (None, 7, 7, 50)          0         \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 2450)              0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 500)               1225500   \n",
      "_________________________________________________________________\n",
      "activation_7 (Activation)    (None, 500)               0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 10)                5010      \n",
      "_________________________________________________________________\n",
      "activation_8 (Activation)    (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 1,256,080\n",
      "Trainable params: 1,256,080\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# create model\n",
    "model = Sequential()\n",
    "\n",
    "# 2 sets of CRP (Convolution, RELU, Pooling)\n",
    "model.add(Conv2D(20, (5, 5),\n",
    "                 padding = \"same\", \n",
    "                 input_shape = input_shape))\n",
    "model.add(Activation(\"relu\"))\n",
    "model.add(MaxPooling2D(pool_size = (2, 2), strides = (2, 2)))\n",
    "\n",
    "model.add(Conv2D(50, (5, 5),\n",
    "                 padding = \"same\"))\n",
    "model.add(Activation(\"relu\"))\n",
    "model.add(MaxPooling2D(pool_size = (2, 2), strides = (2, 2)))\n",
    "\n",
    "# Fully connected layers (w/ RELU)\n",
    "model.add(Flatten())\n",
    "model.add(Dense(500))\n",
    "model.add(Activation(\"relu\"))\n",
    "\n",
    "# Softmax (for classification)\n",
    "model.add(Dense(num_classes))\n",
    "model.add(Activation(\"softmax\"))\n",
    "           \n",
    "model.compile(loss = 'categorical_crossentropy',\n",
    "              optimizer = keras.optimizers.Adadelta(),\n",
    "              metrics = ['accuracy'])\n",
    "    \n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now let us train LeNet on our MNIST Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/10\n",
      "60000/60000 [==============================] - 310s 5ms/step - loss: 0.1855 - acc: 0.9435 - val_loss: 0.0853 - val_acc: 0.9720\n",
      "Epoch 2/10\n",
      "60000/60000 [==============================] - 306s 5ms/step - loss: 0.0449 - acc: 0.9862 - val_loss: 0.0371 - val_acc: 0.9877\n",
      "Epoch 3/10\n",
      "60000/60000 [==============================] - 384s 6ms/step - loss: 0.0298 - acc: 0.9908 - val_loss: 0.0268 - val_acc: 0.9905\n",
      "Epoch 4/10\n",
      "60000/60000 [==============================] - 290s 5ms/step - loss: 0.0207 - acc: 0.9936 - val_loss: 0.0307 - val_acc: 0.9896\n",
      "Epoch 5/10\n",
      "60000/60000 [==============================] - 324s 5ms/step - loss: 0.0153 - acc: 0.9952 - val_loss: 0.0246 - val_acc: 0.9920\n",
      "Epoch 6/10\n",
      "60000/60000 [==============================] - 317s 5ms/step - loss: 0.0110 - acc: 0.9967 - val_loss: 0.0236 - val_acc: 0.9920\n",
      "Epoch 7/10\n",
      "60000/60000 [==============================] - 330s 5ms/step - loss: 0.0088 - acc: 0.9972 - val_loss: 0.0264 - val_acc: 0.9916\n",
      "Epoch 8/10\n",
      "60000/60000 [==============================] - 316s 5ms/step - loss: 0.0069 - acc: 0.9978 - val_loss: 0.0223 - val_acc: 0.9935\n",
      "Epoch 9/10\n",
      "60000/60000 [==============================] - 282s 5ms/step - loss: 0.0052 - acc: 0.9983 - val_loss: 0.0267 - val_acc: 0.9920\n",
      "Epoch 10/10\n",
      "60000/60000 [==============================] - 278s 5ms/step - loss: 0.0037 - acc: 0.9989 - val_loss: 0.0237 - val_acc: 0.9931\n",
      "10000/10000 [==============================] - 28s 3ms/step\n",
      "Test loss: 0.02373578137872546\n",
      "Test accuracy: 0.9931\n"
     ]
    }
   ],
   "source": [
    "# Training Parameters\n",
    "batch_size = 128\n",
    "epochs = 10\n",
    "\n",
    "history = model.fit(x_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=epochs,\n",
    "          validation_data=(x_test, y_test),\n",
    "          shuffle=True)\n",
    "\n",
    "model.save(\"/home/deeplearningcv/DeepLearningCV/Trained Models/mnist_LeNet.h5\")\n",
    "\n",
    "# Evaluate the performance of our trained model\n",
    "scores = model.evaluate(x_test, y_test, verbose=1)\n",
    "print('Test loss:', scores[0])\n",
    "print('Test accuracy:', scores[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}