12.2 Checkpointing Models and Early Stopping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (60000, 28, 28, 1)\n",
      "60000 train samples\n",
      "10000 test samples\n",
      "Number of Classes: 10\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_5 (Conv2D)            (None, 26, 26, 32)        320       \n",
      "_________________________________________________________________\n",
      "conv2d_6 (Conv2D)            (None, 24, 24, 64)        18496     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2 (None, 12, 12, 64)        0         \n",
      "_________________________________________________________________\n",
      "flatten_3 (Flatten)          (None, 9216)              0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 128)               1179776   \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 1,199,882\n",
      "Trainable params: 1,199,882\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "from keras.datasets import mnist\n",
    "from keras.utils import np_utils\n",
    "import keras\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras import backend as K\n",
    "from keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "import os\n",
    "\n",
    "# Training Parameters\n",
    "batch_size = 64\n",
    "epochs = 15\n",
    "\n",
    "# loads the MNIST dataset\n",
    "(x_train, y_train), (x_test, y_test)  = mnist.load_data()\n",
    "\n",
    "# Lets store the number of rows and columns\n",
    "img_rows = x_train[0].shape[0]\n",
    "img_cols = x_train[1].shape[0]\n",
    "\n",
    "# Getting our date in the right 'shape' needed for Keras\n",
    "# We need to add a 4th dimenion to our date thereby changing our\n",
    "# Our original image shape of (60000,28,28) to (60000,28,28,1)\n",
    "x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)\n",
    "x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)\n",
    "\n",
    "# store the shape of a single image \n",
    "input_shape = (img_rows, img_cols, 1)\n",
    "\n",
    "# change our image type to float32 data type\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "\n",
    "# Normalize our data by changing the range from (0 to 255) to (0 to 1)\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "\n",
    "# Now we one hot encode outputs\n",
    "y_train = np_utils.to_categorical(y_train)\n",
    "y_test = np_utils.to_categorical(y_test)\n",
    "\n",
    "# Let's count the number columns in our hot encoded matrix \n",
    "print (\"Number of Classes: \" + str(y_test.shape[1]))\n",
    "\n",
    "num_classes = y_test.shape[1]\n",
    "num_pixels = x_train.shape[1] * x_train.shape[2]\n",
    "\n",
    "# create model\n",
    "model = Sequential()\n",
    "\n",
    "model.add(Conv2D(32, kernel_size=(3, 3),\n",
    "                 activation='relu',\n",
    "                 input_shape=input_shape))\n",
    "model.add(Conv2D(64, (3, 3), activation='relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.25))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(128, activation='relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "model.compile(loss = 'categorical_crossentropy',\n",
    "              optimizer = keras.optimizers.Adadelta(),\n",
    "              metrics = ['accuracy'])\n",
    "\n",
    "print(model.summary())\n",
    "\n",
    "                     \n",
    "checkpoint = ModelCheckpoint(\"/home/deeplearningcv/DeepLearningCV/Trained Models/MNIST_Checkpoint.h5\",\n",
    "                             monitor=\"val_loss\",\n",
    "                             mode=\"min\",\n",
    "                             save_best_only = True,\n",
    "                             verbose=1)\n",
    "callbacks = [checkpoint]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/3\n",
      " - 182s - loss: 0.1478 - acc: 0.9553 - val_loss: 0.0519 - val_acc: 0.9835\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.05185, saving model to /home/deeplearningcv/DeepLearningCV/Trained Models/MNIST_Checkpoint.h5\n",
      "Epoch 2/3\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(x_train, y_train,\n",
    "          batch_size = batch_size,\n",
    "          epochs = epochs,\n",
    "          verbose = 2,\n",
    "          callbacks = callbacks,\n",
    "          validation_data = (x_test, y_test))\n",
    "\n",
    "score = model.evaluate(x_test, y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Adding Multiple Call Backs & Early Stopping\n",
    "\n",
    "We can use other call back methods to monitor our training process such as **Early Stopping**. Checkout the Keras documentation for more:\n",
    "- https://keras.io/callbacks/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.callbacks import EarlyStopping\n",
    "\n",
    "earlystop = EarlyStopping(monitor = 'val_loss', # value being monitored for improvement\n",
    "                          min_delta = 0, #Abs value and is the min change required before we stop\n",
    "                          patience = 3, #Number of epochs we wait before stopping \n",
    "                          verbose = 1,\n",
    "                          restore_best_weights = True) #keeps the best weigths once stopped\n",
    "\n",
    "# we put our call backs into a callback list\n",
    "callbacks = [earlystop, checkpoint]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### We can attempt to run again to see if it worked!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "history = model.fit(x_train, y_train,\n",
    "          batch_size=64,\n",
    "          epochs=3,\n",
    "          verbose=2,\n",
    "          callbacks = callbacks,\n",
    "          validation_data=(x_test, y_test))\n",
    "\n",
    "\n",
    "score = model.evaluate(x_test, y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Another useful callback is Reducing our learning Rate on Plateau\n",
    "\n",
    "We can avoid having our oscillate around the global minimum by attempting to reduce the Learn Rate by a certain fact. If no improvement is seen in our monitored metric (val_loss typically), we wait a certain number of epochs (patience) then this callback reduces the learning rate by a factor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.callbacks import ReduceLROnPlateau\n",
    "\n",
    "reduce_lr = ReduceLROnPlateau(monitor = 'val_loss', factor = 0.2, patience = 3, verbose = 1, min_delta = 0.0001)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}