11.1 - 11.2 - MNIST Confusion Matrix Analysis and Viewing Misclassifications.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Confusion Matrix Analysis and Viewing Misclassifications"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (60000, 28, 28, 1)\n",
      "60000 train samples\n",
      "10000 test samples\n",
      "Number of Classes: 10\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_3 (Conv2D)            (None, 26, 26, 32)        320       \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 24, 24, 64)        18496     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 12, 12, 64)        0         \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 12, 12, 64)        0         \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 9216)              0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 128)               1179776   \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 1,199,882\n",
      "Trainable params: 1,199,882\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/1\n",
      "60000/60000 [==============================] - 147s 2ms/step - loss: 0.2660 - acc: 0.9171 - val_loss: 0.0604 - val_acc: 0.9811\n",
      "Test loss: 0.060351789912581444\n",
      "Test accuracy: 0.9811\n"
     ]
    }
   ],
   "source": [
    "from keras.datasets import mnist\n",
    "from keras.utils import np_utils\n",
    "import keras\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras import backend as K\n",
    "\n",
    "# Training Parameters\n",
    "batch_size = 128\n",
    "epochs = 1\n",
    "\n",
    "# loads the MNIST dataset\n",
    "(x_train, y_train), (x_test, y_test)  = mnist.load_data()\n",
    "\n",
    "# Lets store the number of rows and columns\n",
    "img_rows = x_train[0].shape[0]\n",
    "img_cols = x_train[1].shape[0]\n",
    "\n",
    "# Getting our date in the right 'shape' needed for Keras\n",
    "# We need to add a 4th dimenion to our date thereby changing our\n",
    "# Our original image shape of (60000,28,28) to (60000,28,28,1)\n",
    "x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)\n",
    "x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)\n",
    "\n",
    "# store the shape of a single image \n",
    "input_shape = (img_rows, img_cols, 1)\n",
    "\n",
    "# change our image type to float32 data type\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "\n",
    "# Normalize our data by changing the range from (0 to 255) to (0 to 1)\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "\n",
    "# Now we one hot encode outputs\n",
    "y_train = np_utils.to_categorical(y_train)\n",
    "y_test = np_utils.to_categorical(y_test)\n",
    "\n",
    "# Let's count the number columns in our hot encoded matrix \n",
    "print (\"Number of Classes: \" + str(y_test.shape[1]))\n",
    "\n",
    "num_classes = y_test.shape[1]\n",
    "num_pixels = x_train.shape[1] * x_train.shape[2]\n",
    "\n",
    "# create model\n",
    "model = Sequential()\n",
    "\n",
    "model.add(Conv2D(32, kernel_size=(3, 3),\n",
    "                 activation='relu',\n",
    "                 input_shape=input_shape))\n",
    "model.add(Conv2D(64, (3, 3), activation='relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.25))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(128, activation='relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "model.compile(loss = 'categorical_crossentropy',\n",
    "              optimizer = keras.optimizers.Adadelta(),\n",
    "              metrics = ['accuracy'])\n",
    "\n",
    "print(model.summary())\n",
    "\n",
    "\n",
    "history = model.fit(x_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=epochs,\n",
    "          verbose=1,\n",
    "          validation_data=(x_test, y_test))\n",
    "\n",
    "score = model.evaluate(x_test, y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Let's save our history file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle \n",
    "\n",
    "pickle_out = open(\"MNIST_history.pickle\",\"wb\")\n",
    "pickle.dump(history.history, pickle_out)\n",
    "pickle_out.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Loading out saved history is as simple as these two lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'val_loss': [0.07168159085139632], 'val_acc': [0.9774], 'loss': [0.26640635084311165], 'acc': [0.9185166666348775]}\n"
     ]
    }
   ],
   "source": [
    "pickle_in = open(\"MNIST_history.pickle\",\"rb\")\n",
    "saved_history = pickle.load(pickle_in)\n",
    "print(saved_history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3X2cVnWd//HX22FwDBARlJRxhQ1aAR2HcQLNQPAGsf2FN2sK3qIWZlm2PizRWm1Z+62VqdlqKxqUt6zpGuwCIiFT+SsLUBxFHggi6SCmWCoDiI18fn9cB7wYh7nhzJlrhnk/H4/z4Fzfc77f6/OB0c+cu+9RRGBmZra79ip0AGZm1rG5kJiZWSouJGZmlooLiZmZpeJCYmZmqbiQmJlZKi4kZmaWSqaFRNI4SSslrZY0pYHtV0p6QVK1pIWSDk3ax0halre8J+m0ZNvPJL2ct608yxzMzKxxyuqBRElFwIvASUANsBiYGBEv5O0zBvhDRGyWdBkwOiLOrjfO/sBqoDTZ72fA/0bEw5kEbmZmLdIlw7GHA6sjYg2ApJnAqcCOQhIRi/L2fwo4r4FxzgTmRcTm3Q2kT58+0b9//93tXhCbNm2iW7duhQ6jTTnnzsE5dxxLly7dEBEHNLVfloWkH/Bq3ucaYEQj+18CzGugfQJwc72270q6DlgITImIrY0F0r9/f5YsWdJ0xO1IVVUVo0ePLnQYbco5dw7OueOQ9Kfm7JdlIWk2SecBlcBx9doPAo4A5uc1XwO8DnQFpgFXA1MbGHMyMBmgb9++VFVVZRF6ZmpraztczGk5587BOe95siwk64BD8j6XJm07kXQi8C3guAaOLM4CHo2Iv21viIj1yepWSTOAqxr68oiYRq7QUFlZGR3tt4GO+htMGs65c3DOe54s79paDAySNEBSV3KnqGbn7yBpGHAnMD4i3mhgjInAg/X6HJT8KeA04PkMYjczs2bK7IgkIuokXU7utFQRMD0ilkuaCiyJiNnAD4DuwC9ydYFXImI8gKT+5I5ofl1v6PslHQAIWAZ8KasczPYkf/vb36ipqeG9994raBw9e/ZkxYoVBY2hrbX3nEtKSigtLaW4uHi3+md6jSQi5gJz67Vdl7d+YiN915K7YF+//fhWDNGsMJ68BY48p02/sqamhh49etC/f3+SX9wKYuPGjfTo0aNg318I7TnniOCtt96ipqaGAQMG7NYYfrLdrBBq34D/96M2/cr33nuP3r17F7SIWPsjid69e6c6UnUhMSuEY6+AZx+g69a/tunXuohYQ9L+XLiQmBVCj4/DkRM55NX/LnQkTbplwYuFDsHaORcSs0L55MkcUjMbvtNz52XRvxc6sp38aOGqVhlnzJgxzJ8/f6e2W2+9lcsuu6zRft27dwfgtdde48wzz2xwn9GjRzf50PGtt97K5s0fTpDx2c9+lrfffrs5oTfoqaeeYsCAAZSXl1NeXk737t35h3/4B8rLy7ngggtaNNa2bdu48cYbd7m9tLQ0VaxZcyExK4T3N8Hcb7LisCvgO+/svIy5ptDRZWLixInMnDlzp7aZM2cyceLEZvU/+OCDefjh3Z9ir34hmTt3Lvvtt99ujzdv3jxuuukmli1bxrJly6isrOT+++9n2bJl3HPPPS0aq6lC0t65kJgVwpyroLSSP3+889yEeOaZZzJnzhzef/99ANauXctrr73GyJEjqa2t5YQTTqCiooIjjjiCWbNmfaT/2rVrOfzwwwHYsmULEyZMYPDgwZx++uls2bJlx36XXXYZlZWVDB06lOuvvx6A2267jddee40xY8YwZswYIDd10oYNGwC4+eabOfzwwzn88MO59dZbd3zf4MGD+eIXv8jQoUMZO3bsTt+zcOFCTjxxlzeeUldXx5VXXsnw4cM55phjuPvuuwFYt24dn/nMZygvL+fwww/nd7/7HVOmTGHjxo0tOprZsGED48ePp6ysjE9/+tM8/3zukbonnniCI488kvLycioqKti0aVOD39ma2sUUKWadyjP3w2tPwxefgN8tLkgI/afMyWT/tTf+4y637b///gwfPpwFCxYwYcIEZs6cyVlnnYUkSkpKePTRR9l3333ZsGEDRx99NOPHj9/lReCf/OQnfOxjH2PFihVUV1dTUVGxY9t3v/td9t9/fz744ANOOOEEqqur+drXvsbNN9/MokWL6NOnz05jLV26lBkzZvCHP/yBiGDEiBEcd9xx9OrVi1WrVvHggw9y1113cdZZZ/HII49w3nnnsWHDBoqLi+nZs+cu8502bRoHHnggf/zjH9mwYQMnnXQSY8eO5cEHH+Rzn/scV199NR988AFbtmxh+PDh3H333SxbtqxZf88A//Iv/8KIESOYPXs2jz/+OJMmTWLJkiX84Ac/YNq0aYwYMYLa2lpKSkq47777PvKdrclHJGZtbdOb8PmfQdeONxtsWhMnTtxxeir/tFZEcO2111JWVsaJJ57IunXr+POf/7zLcX7zm99w3nm5ycLLysooKyvbse2hhx6ioqKCYcOGsXz5cl544YVdDQPAk08+yemnn063bt3o3r07Z5xxBr/97W8BdlwDATjqqKNYu3YtAI8//jhjx45tdNzHH3+cGTNmUF5ezvHHH8/bb7/NqlWr+NSnPsXdd9/Nv/7rv/L888/vuAbUUk8++STnn38+AGPHjuW1115j06ZNHHvssVxxxRX8+Mc/5t1336WoqKjVvnNXfERi1tY+8/VCR9DokUN9/afMadH+jTn11FP5+te/ztNPP83mzZs56qijALj//vt58803Wbp0KcXFxfTv33+3nmt4+eWXuemmm1i8eDG9evVi0qRJqZ6P2HvvvXesFxUV7fhNft68eVx55ZWN9o0I7rjjDk444YSPPJBYVVXFnDlzuOCCC/jmN7/J2Wef3chILfPtb3+b8ePHM2fOHI4++mgWLlzI8ccf/5HvPPfcc1vtO31EYmZtpnv37owaNYqLL754p4vs77zzDgceeCDFxcUsWrSIP/2p8dnLR40axQMPPADA888/T3V1NQDvvvsu3bp1o2fPnvz5z39m3rwP30zRo0cPNm7c+JGxRo4cyS9/+Us2b97Mpk2bePTRRxk5cuQuvzsiqK6u3nGksisnn3wyd9xxB3V1dQCsXLmSLVu28Kc//YmPf/zjTJ48mYsuuohnnnmGLl1yv9Nv37c5Ro4cyf333w/Ar371K/r160e3bt146aWXKCsr45prrqGiooKVK1c2+J2tyUckZtamzjzzTM4555yd7uA699xz+dznPscRRxxBZWUlhx12WKNjXHbZZVx00UUMHjyYwYMH7ziyOfLIIxk2bBiHHXYYhxxyCMcee+yOPpMnT2bcuHEcfPDBLFr04Tv1KioqmDRpEsOHDwfgC1/4AsOGDdtxGqu+pUuXMmzYsCYf4rv00kt55ZVXKC8vZ9u2bXz84x9n1qxZLFy4kJtvvpni4mJ69OjBvffeC8All1xCWVkZlZWVDd71NXTo0B3fec455zB16lQuvvhiysrK6N69OzNmzADgpptu4re//S177bUXZWVljB07lvvuu6/B72wtmb1qtz2prKwMv9iq/XPO2VqxYgWDBw9ucb/WPLUF7Xveqea44YYbGDhwIBMmTGh2n46Qc0M/H5KWRkRlU319RGJmjbrihEGFDqFd+fa3v13oENodXyMxs0b980mfLHQI1s65kJiZWSouJGZmlkqmhUTSOEkrJa2WNKWB7VdKekFStaSFkg7N2/aBpGXJMjuvfYCkPyRj/lfyGl8zMyuQzAqJpCLgduAUYAgwUdKQers9A1RGRBnwMPD9vG1bIqI8WcbntX8PuCUiBgJ/BS7JKgczM2talkckw4HVEbEmIt4HZgKn5u8QEYsiYvt0nE8BpY0NqNxN1MeTKzoAPwdOa9WozSznyVtg466nKWmJt956a8d06wMHDqRfv347Pm+fxLEpF110EStXrmx0n9tvv33HQ3ppfeYzn2nR3FedWZa3//YDXs37XAOMaGT/S4B5eZ9LJC0B6oAbI+KXQG/g7YjY/vhnDQ28193MWsH21wGP+7+ph+rdu/eO/ylfc8019O7dm6uuumqnfSKCiGCvvRr+/Xb7A3eN+cpXvpI6Vmu5dvEciaTzgErguLzmQyNinaS/B56Q9BzwTgvGnAxMBujbty9VVVWtGHH2amtrO1zMaTnnbPXs2bPBKUJ2RUdeQrefn8Cm8kuIbge2WhwRwdatW9m4cSMvvfQSEyZMoKysjOrqambNmsWNN97Is88+y5YtWzjjjDOYMiV3eXXs2LHcdNNNDBkyhAEDBnDxxRezYMEC9tlnH2bOnMkBBxzA1KlT6d27N1/5ylcYO3YsxxxzDL/+9a959913+clPfsKIESPYtGkTl156KStXruSwww7jlVde4cc//vFOEz8CfPDBB2zatGmnv7MtW7bw9a9/nWeffZbi4mJuvPFGjj32WJYvX86Xv/xl6urq2LZtGw888AB9+vThwgsv5PXXX6euro5rr72W005rvydQ3nvvvd3+WcyykKwDDsn7XJq07UTSicC3gOMiYuv29ohYl/y5RlIVMAx4BNhPUpfkqKTBMZN+04BpkHuyvaM9Me2nvDuHtn6yfcfT1d/Z9fTn9XX/z4qmd9ruO03/rieJvffemx49etC9e3defPFF7rvvPiorcw9Q//CHP2T//fenrq6OMWPGcO655zJkyBCKioro1q0bPXr04J133uGkk07i5ptv5sorr+Shhx5iypQp7L333pSUlNCjRw+KioooLi5m6dKlzJ49m5tuuonHHnuM22+/ndLSUmbNmsWzzz5LRUXFjnHz5X/fdnfccQfdu3dn+fLlLF++nM9+9rOsWrWKn//851x99dWcffbZbN26lYhg1qxZDBw4kAULFrBx40a2bdvWrp9uLykpYdiwYbvVN8trJIuBQcldVl2BCcDs/B0kDQPuBMZHxBt57b0k7Z2s9wGOBV6I3Hwui4Dt79u8EPjoG3DMrMP4xCc+saOIADz44INUVFRQUVHBihUrGpwGfp999uGUU04Bdp7evb4zzjjjI/s8+eSTO6Y3OfLIIxk6dGizY33yySd3TF8/dOhQDj74YFavXs2nP/1pbrjhBr7//e/z6quvUlJSQllZGY899hhTpkzhqaeeavTdJR1dZkckEVEn6XJgPlAETI+I5ZKmAksiYjbwA6A78ItkMrJXkju0BgN3StpGrtjdGBHbf5quBmZKuoHcXV8/zSoHsz1WM44ceH8TTBuTm/a+/JzMQunW7cP3sqxatYof/ehH/PGPf2S//fbjvPPOa3Aa+K5dP7zrv6ioaJez5m6fBr6xfVrD+eefzzHHHMOcOXMYN24c06dPZ9SoUSxZsoS5c+dy/fXXs3jxYq699trMYiikTK+RRMRcYG69tuvy1ht8T2VE/A44Yhfb1pC7I8zMspS8DjjLIlLfu+++S48ePdh3331Zv3498+fPZ9y4ca36HcceeywPPfQQI0eO5LnnnmvyxVf5tk/dPmrUKFasWMH69esZOHAga9asYeDAgVxxxRW8/PLLVFdX84lPfII+ffpw/vnn06VLFx555JFWzaM9aRcX282sncl/HXAbqqioYMiQIRx22GEceuihO00D31q++tWvcsEFFzBkyJAdy65OO5188skUFxcDuSIyffp0Lr30Uo444giKi4u555576Nq1Kw888AAPPvggxcXFHHzwwXznO9/Z8S72vfbai6KiIu66665Wz6W98DTy7ZQvPHcO7XYa+SdvhU+eDAe2fNr5phR6SvW6ujrq6uooKSlh1apVjB07llWrVu14uVQWCp1zc3gaeTNrXe3gdcBZqa2t5YQTTqCuro6I4M4778y0iHQG/tszs05lv/32Y+nSpYUOY4/i2X/NOpHOcCrbWi7tz4ULiVknUVJSwltvveViYjuJCN566y1KSkp2ewyf2jLrJEpLS6mpqeHNN98saBzvvfdeqv9pdUTtPeeSkhJKSxudM7dRLiRmnURxcTEDBgwodBhUVVXt9lQcHdWenrNPbZmZWSouJGZmlooLiZmZpeJCYmZmqbiQmJlZKi4kZmaWiguJmZml4kJiZmapuJCYmVkqmRYSSeMkrZS0WtKUBrZfKekFSdWSFko6NGkvl/R7ScuTbWfn9fmZpJclLUuW8ixzMDOzxmVWSCQVAbcDpwBDgImShtTb7RmgMiLKgIeB7yftm4ELImIoMA64VdJ+ef2+ERHlybIsqxzMzKxpWR6RDAdWR8SaiHgfmAmcmr9DRCyKiM3Jx6eA0qT9xYhYlay/BrwBHJBhrGZmtpuynLSxH/Bq3ucaYEQj+18CzKvfKGk40BV4Ka/5u5KuAxYCUyJiawP9JgOTAfr27UtVVVVL4y+o2traDhdzWs65c3DOe552MfuvpPOASuC4eu0HAfcCF0bEtqT5GuB1csVlGnA1MLX+mBExLdlOZWVldLR3gfv95Z2Dc+4c9vScszy1tQ44JO9zadK2E0knAt8CxucfWUjaF5gDfCsintreHhHrI2crMIPcKTQzMyuQLAvJYmCQpAGSugITgNn5O0gaBtxJroi8kdfeFXgUuCciHq7X56DkTwGnAc9nmIOZmTUhs1NbEVEn6XJgPlAETI+I5ZKmAksiYjbwA6A78ItcXeCViBgPnAWMAnpLmpQMOSm5Q+t+SQcAApYBX8oqBzMza1qm10giYi4wt17bdXnrJ+6i333AfbvYdnxrxmhmZun4yXYzM0vFhcTMzFJxITEzs1RcSMzMLBUXEjMzS8WFxMzMUnEhMTOzVFxIzMwsFRcSMzNLxYXEzMxScSExM7NUXEjMzCwVFxIzM0vFhcTMzFJxITEzs1RcSMzMLJVMC4mkcZJWSlotaUoD26+U9IKkakkLJR2at+1CSauS5cK89qMkPZeMeVvyyl0zMyuQzAqJpCLgduAUYAgwUdKQers9A1RGRBnwMPD9pO/+wPXACGA4cL2kXkmfnwBfBAYly7iscjAzs6ZleUQyHFgdEWsi4n1gJnBq/g4RsSgiNicfnwJKk/WTgQUR8ZeI+CuwABgn6SBg34h4KiICuAc4LcMczMysCVkWkn7Aq3mfa5K2XbkEmNdE337JenPHNDOzjHUpdAAAks4DKoHjWnHMycBkgL59+1JVVdVaQ7eJ2traDhdzWs65c3DOe54sC8k64JC8z6VJ204knQh8CzguIrbm9R1dr29V0l5ar/0jYwJExDRgGkBlZWWMHj26od3araqqKjpazGk5587BOe95sjy1tRgYJGmApK7ABGB2/g6ShgF3AuMj4o28TfOBsZJ6JRfZxwLzI2I98K6ko5O7tS4AZmWYg5mZNSGzI5KIqJN0ObmiUARMj4jlkqYCSyJiNvADoDvwi+Qu3lciYnxE/EXSv5ErRgBTI+IvyfqXgZ8B+5C7pjIPMzMrmEyvkUTEXGBuvbbr8tZPbKTvdGB6A+1LgMNbMUwzM0vBT7abmVkqLiRmZpaKC4mZmaXiQmJmZqm4kJiZWSouJGZmlooLiZmZpeJCYmZmqbiQmJlZKi4kZmaWiguJmZml4kJiZmapuJCYmVkqLiRmZpaKC4mZmaXiQmJmZqlkWkgkjZO0UtJqSVMa2D5K0tOS6iSdmdc+RtKyvOU9Sacl234m6eW8beVZ5mBmZo1r1hsSJX0CqImIrZJGA2XAPRHxdiN9ioDbgZOAGmCxpNkR8ULebq8Ak4Cr8vtGxCKgPBlnf2A18HjeLt+IiIebE7uZmWWruUckjwAfSBoITAMOAR5oos9wYHVErImI94GZwKn5O0TE2oioBrY1Ms6ZwLyI2NzMWM3MrA01t5Bsi4g64HTgxxHxDeCgJvr0A17N+1yTtLXUBODBem3flVQt6RZJe+/GmGZm1kqadWoL+JukicCFwOeStuJsQvqQpIOAI4D5ec3XAK8DXckdHV0NTG2g72RgMkDfvn2pqqrKOtxWVVtb2+FiTss5dw7Oec/T3EJyEfAl4LsR8bKkAcC9TfRZR+4U2HalSVtLnAU8GhF/294QEeuT1a2SZlDv+kreftPIFRoqKytj9OjRLfzqwqqqqqKjxZyWc+4cnPOep1mFJLlA/jUASb2AHhHxvSa6LQYGJUVnHblTVOe0ML6J5I5AdpB0UESslyTgNOD5Fo5pZmatqFnXSCRVSdo3uYPqaeAuSTc31ie5pnI5udNSK4CHImK5pKmSxifjfkpSDfB54E5Jy/O+sz+5I5pf1xv6fknPAc8BfYAbmpODmZllo7mntnpGxLuSvkDutt/rJVU31Ski5gJz67Vdl7e+mNwpr4b6rqWBi/MRcXwzYzYzszbQ3Lu2uiQXvs8C/jfDeMzMrINpbiGZSu4U1UsRsVjS3wOrsgvLzMw6iuZebP8F8Iu8z2uAf8oqKDMz6ziae7G9VNKjkt5IlkckNXhtw8zMOpfmntqaAcwGDk6W/0nazMysk2tuITkgImZERF2y/Aw4IMO4zMysg2huIXlL0nmSipLlPOCtLAMzM7OOobmF5GJyt/6+DqwnNyPvpIxiMjOzDqRZhSQi/hQR4yPigIg4MCJOw3dtmZkZ6d6QeGWrRWFmZh1WmkKiVovCzMw6rDSFJFotCjMz67AafbJd0kYaLhgC9skkIjMz61AaLSQR0aOtAjEzs44pzaktMzMzFxIzM0vHhcTMzFLJtJBIGidppaTVkqY0sH2UpKcl1Uk6s962DyQtS5bZee0DJP0hGfO/JHXNMgczM2tcZoVEUhFwO3AKMASYKGlIvd1eITfVygMNDLElIsqTZXxe+/eAWyJiIPBX4JJWD97MzJotyyOS4cDqiFgTEe8DM4FT83eIiLURUQ1sa86AkgQcDzycNP0cOK31QjYzs5Zq1hsSd1M/4NW8zzXAiBb0L5G0BKgDboyIXwK9gbcjoi5vzH4NdZY0GZgM0LdvX6qqqloWfYHV1tZ2uJjTcs6dg3Pe82RZSNI6NCLWJe+Hf0LSc8A7ze0cEdOAaQCVlZUxevTobKLMSFVVFR0t5rScc+fgnPc8WZ7aWgcckve5NGlrlohYl/y5BqgChpF7B8p+krYXwBaNaWZmrS/LQrIYGJTcZdUVmEDudb1NktRL0t7Jeh/gWOCFiAhgEbn3oQBcCMxq9cjNzKzZMiskyXWMy4H5wArgoYhYLmmqpPEAkj4lqQb4PHCnpOVJ98HAEknPkiscN0bEC8m2q4ErJa0md83kp1nlYGZmTcv0GklEzAXm1mu7Lm99MbnTU/X7/Q44YhdjriF3R5iZmbUDfrLdzMxScSExM7NUXEjMzCwVFxIzM0vFhcTMzFJxITEzs1RcSMzMLBUXEjMzS8WFxMzMUnEhMTOzVFxIzMwsFRcSMzNLxYXEzMxScSExM7NUXEjMzCwVFxIzM0sl00IiaZyklZJWS5rSwPZRkp6WVCfpzLz2ckm/l7RcUrWks/O2/UzSy5KWJUt5ljmYmVnjMntDoqQi4HbgJKAGWCxpdt4rcwFeASYBV9Xrvhm4ICJWSToYWCppfkS8nWz/RkQ8nFXsZmbWfFm+anc4sDp5NS6SZgKnAjsKSUSsTbZty+8YES/mrb8m6Q3gAOBtzMysXcny1FY/4NW8zzVJW4tIGg50BV7Ka/5ucsrrFkl7pwvTzMzSyPKIJDVJBwH3AhdGxPajlmuA18kVl2nA1cDUBvpOBiYD9O3bl6qqqrYIudXU1tZ2uJjTcs6dg3Pe82RZSNYBh+R9Lk3amkXSvsAc4FsR8dT29ohYn6xulTSDj15f2b7fNHKFhsrKyhg9enSLgi+0qqoqOlrMaTnnzsE573myPLW1GBgkaYCkrsAEYHZzOib7PwrcU/+ienKUgiQBpwHPt2rUZmbWIpkVkoioAy4H5gMrgIciYrmkqZLGA0j6lKQa4PPAnZKWJ93PAkYBkxq4zfd+Sc8BzwF9gBuyysHMzJqW6TWSiJgLzK3Xdl3e+mJyp7zq97sPuG8XYx7fymGamVkKfrLdzMxScSExM7NUXEjMzCwVFxIzM0vFhcTMzFJxITEzs1RcSMzMLBUXEjMzS8WFxMzMUnEhMTOzVFxIzMwsFRcSMzNLxYXEzMxScSExM7NUXEjMzCwVFxIzM0vFhcTMzFLJtJBIGidppaTVkqY0sH2UpKcl1Uk6s962CyWtSpYL89qPkvRcMuZtybvbzcysQDIrJJKKgNuBU4AhwERJQ+rt9gowCXigXt/9geuBEcBw4HpJvZLNPwG+CAxKlnEZpWBmZs2Q5RHJcGB1RKyJiPeBmcCp+TtExNqIqAa21et7MrAgIv4SEX8FFgDjJB0E7BsRT0VEAPcAp2WYg5mZNaFLhmP3A17N+1xD7ghjd/v2S5aaBto/QtJkYDJA3759qaqqauZXtw+1tbUdLua0nHPn4Jz3PFkWkoKKiGnANIDKysoYPXp0YQNqoaqqKjpazGk5587BOe95sjy1tQ44JO9zadKWpu+6ZH13xjQzswxkWUgWA4MkDZDUFZgAzG5m3/nAWEm9kovsY4H5EbEeeFfS0cndWhcAs7II3szMmiezQhIRdcDl5IrCCuChiFguaaqk8QCSPiWpBvg8cKek5UnfvwD/Rq4YLQamJm0AXwbuBlYDLwHzssrBzMyaluk1koiYC8yt13Zd3vpidj5Vlb/fdGB6A+1LgMNbN1IzM9tdfrLdzMxScSExM7NUXEjMzCwVFxIzM0vFhcTMzFJxITEzs1RcSMzMLBUXEjMzS8WFxMzMUnEhMTOzVFxIzMwsFRcSMzNLxYXEzMxScSExM7NUXEjMzCwVFxIzM0sl00IiaZyklZJWS5rSwPa9Jf1Xsv0Pkvon7edKWpa3bJNUnmyrSsbcvu3ALHMwM7PGZVZIJBUBtwOnAEOAiZKG1NvtEuCvETEQuAX4HkBE3B8R5RFRDpwPvBwRy/L6nbt9e0S8kVUOZmbWtCyPSIYDqyNiTUS8D8wETq23z6nAz5P1h4ETJKnePhOTvmZ7nEdXvV/oEMxSy7KQ9ANezftck7Q1uE9E1AHvAL3r7XM28GC9thnJaa1/aaDwmHUYs176W6FDMEutS6EDaIykEcDmiHg+r/nciFjDLzOZAAAHY0lEQVQnqQfwCLlTX/c00HcyMBmgb9++VFVVtUHErae2trbDxZxWZ8wZ6HQ5d8Z/5z095ywLyTrgkLzPpUlbQ/vUSOoC9ATeyts+gXpHIxGxLvlzo6QHyJ1C+0ghiYhpwDSAysrKGD16dJpc2lxVVRUdLea0OmPOPDan0+XcGf+d9/Scszy1tRgYJGmApK7kisLsevvMBi5M1s8EnoiIAJC0F3AWeddHJHWR1CdZLwb+D/A8ZmZWMJkdkUREnaTLgflAETA9IpZLmgosiYjZwE+BeyWtBv5CrthsNwp4NSLW5LXtDcxPikgR8CvgrqxyMGtNtyx4kR8tXPWR9v5T5uz0+YoTBvHPJ32yrcIySy3TayQRMReYW6/turz194DP76JvFXB0vbZNwFGtHqhZG/jnkz75kQLRf8oc1t74jwWKyKx1+Ml2MzNLxYXEzMxScSExM7NUXEjMCujUTxQXOgSz1FxIzAro9EFdCx2CWWouJGZmlooLiZmZpeJCYmZmqSiZkWSPJulN4E+FjqOF+gAbCh1EG3POnYNz7jgOjYgDmtqpUxSSjkjSkoioLHQcbck5dw7Oec/jU1tmZpaKC4mZmaXiQtJ+TSt0AAXgnDsH57yH8TUSMzNLxUckZmaWigtJAUgaJ2mlpNWSpjSw/VBJCyVVS6qSVJq37e8kPS5phaQXJPVvy9h3V8qcvy9peZLzbZLUttG3nKTpkt6Q1OAbPJVzW/L3US2pIm/bhZJWJcuFDfVvj3Y3Z0nlkn6f/BtXSzq7bSPffWn+nZPt+0qqkfQfbRNxRiLCSxsu5N7s+BLw90BX4FlgSL19fgFcmKwfD9ybt60KOClZ7w58rNA5ZZkz8Gng/yVjFAG/B0YXOqdm5DwKqACe38X2zwLzAJF7gdsfkvb9gTXJn72S9V6FzifjnD8JDErWDwbWA/sVOp8sc87b/iPgAeA/Cp1LmsVHJG1vOLA6ItZExPvk3kl/ar19hgBPJOuLtm+XNAToEhELACKiNiI2t03Yqex2zkAAJeQK0N5AMfDnzCNOKSJ+Q+710btyKnBP5DwF7CfpIOBkYEFE/CUi/gosAMZlH3F6u5tzRLwYEauSMV4D3gCafAiuPUjx74yko4C+wOPZR5otF5K21w94Ne9zTdKW71ngjGT9dKCHpN7kfnN7W9J/S3pG0g8kFWUecXq7nXNE/J5cYVmfLPMjYkXG8baFXf2dNOfvqqNqMjdJw8n90vBSG8aVpQZzlrQX8EPgqoJE1cpcSNqnq4DjJD0DHAesAz4AugAjk+2fIneqaFKBYmxtDeYsaSAwGCgl9x/l8ZJGFi5My0rym/q9wEURsa3Q8WTsy8DciKgpdCCtoUuhA+iE1gGH5H0uTdp2SA7vzwCQ1B34p4h4W1INsCwi1iTbfknuvOtP2yLwFNLk/EXgqYioTbbNA44BftsWgWdoV38n64DR9dqr2iyqbO3y50DSvsAc4FvJKaA9xa5yPgYYKenL5K51dpVUGxEfuRGlI/ARSdtbDAySNEBSV2ACMDt/B0l9kkNfgGuA6Xl995O0/fzx8cALbRBzWmlyfoXckUoXScXkjlb2hFNbs4ELkrt6jgbeiYj1wHxgrKReknoBY5O2PUGDOSc/E4+Su5bwcGFDbHUN5hwR50bE30VEf3JH4/d01CICPiJpcxFRJ+lycv9zKAKmR8RySVOBJRExm9xvpP8uKYDfAF9J+n4g6SpgYXIL7FLgrkLk0RJpcgYeJlcwnyN34f2xiPifts6hpSQ9SC6nPsmR5PXkbhQgIv4TmEvujp7VwGbgomTbXyT9G7niCzA1Ihq7mNtu7G7OwFnk7n7qLWlS0jYpIpa1WfC7KUXOexQ/2W5mZqn41JaZmaXiQmJmZqm4kJiZWSouJGZmlooLiZmZpeJCYrabJH0gaVne0mrPAUjqv6sZZc3aGz9HYrb7tkREeaGDMCs0H5GYtTJJa5V7h8pzkv6YzBe2/SjjieS9FAsl/V3S3lfSo5KeTZZPJ0MVSboreU/H45L2Sfb/mnLvoqmWNLNAaZrt4EJitvv2qXdqK/+FTO9ExBHAfwC3Jm0/Bn4eEWXA/cBtSfttwK8j4khy77ZYnrQPAm6PiKHA28A/Je1TgGHJOF/KKjmz5vKT7Wa7KZlkr3sD7WuB4yNiTTI/2OsR0VvSBuCgiPhb0r4+IvpIehMojYiteWP0J/dekkHJ56uB4oi4QdJjQC3wS+CX2ye0NCsUH5GYZSN2sd4SW/PWt79GAOAfgdvJHb0sluRrnVZQLiRm2Tg778/fJ+u/IzfzMcC5fDgV/kLgMgBJRZJ67mrQZIbkQyJiEXA10JPcNORmBePfZMx23z6S8meofSxvKvBekqrJHVVMTNq+CsyQ9A3gTT6cCfYKYJqkS8gdeVxG7m2QDSkC7kuKjYDbIuLtVsvIbDf4GolZK0uukVRGxIZCx2LWFnxqy8zMUvERiZmZpeIjEjMzS8WFxMzMUnEhMTOzVFxIzMwsFRcSMzNLxYXEzMxS+f+oM1e5gk7CIwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plotting our loss charts\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "history_dict = history.history\n",
    "\n",
    "loss_values = history_dict['loss']\n",
    "val_loss_values = history_dict['val_loss']\n",
    "epochs = range(1, len(loss_values) + 1)\n",
    "\n",
    "line1 = plt.plot(epochs, val_loss_values, label='Validation/Test Loss')\n",
    "line2 = plt.plot(epochs, loss_values, label='Training Loss')\n",
    "plt.setp(line1, linewidth=2.0, marker = '+', markersize=10.0)\n",
    "plt.setp(line2, linewidth=2.0, marker = '4', markersize=10.0)\n",
    "plt.xlabel('Epochs') \n",
    "plt.ylabel('Loss')\n",
    "plt.grid(True)\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3Xt8FdW5//HPQwCxBgGhhtZQwBaVgOEWQQQkkQro+VXKRQXxWLSKpaBWX6ho/VlL6xFPrVULaqlFpUeDFBU5P0BQJAWPd6pggSLIQQ0gCsglXJTA8/tjT7abmGRCksnO5ft+vfYrM2vWzH5WAnmy1pq9xtwdERGRsjRIdgAiIlLzKVmIiEgoJQsREQmlZCEiIqGULEREJJSShYiIhFKyEBGRUJEmCzMbbGbrzGyDmU0q4XhbM1tiZqvMLM/M0hOO/aeZrTaztWb2kJlZlLGKiEjpIksWZpYCTAMuADKAUWaWUazafcBMd88EJgP3BOeeA/QBMoHOwFlA/6hiFRGRsjWM8No9gQ3uvhHAzGYBQ4A1CXUygJuC7aXA3GDbgSZAY8CARsC2st6sVatW3q5du6qKvdrs27ePE044IdlhVCu1uX5Qm2uHFStWbHf3b4fVizJZnAJ8krCfD/QqVmclMAx4EBgKNDWzlu7+upktBbYSSxZT3X1tWW/Wrl073nnnnSoLvrrk5eWRnZ2d7DCqldpcP6jNtYOZfVSeelEmi/KYCEw1szHAMmAzcNjMfgB0BIrmMF4ys37uvjzxZDMbC4wFSEtLIy8vr7rirjIFBQW1Mu7KUJvrB7W5bokyWWwG2iTspwdlce6+hVjPAjNLBYa7+y4zuwZ4w90LgmMLgd7A8mLnTwemA2RlZXlty+hQO/8SqSy1uX5Qm+uWKO+GehvoYGbtzawxMBKYl1jBzFqZWVEMtwEzgu2Pgf5m1tDMGhGb3C5zGEpERKITWc/C3QvNbAKwCEgBZrj7ajObDLzj7vOAbOAeM3Niw1Djg9PnAOcB7xOb7H7R3f87qlhFovb8+q/IzoZDhw6Rn5/PwYMHkx1S5Jo1a8batfXrb7ya3OYmTZqQnp5Oo0aNKnR+pHMW7r4AWFCs7M6E7TnEEkPx8w4D10YZm0h1euHDQzwI5Ofn07RpU9q1a0dd/+jQ3r17adq0abLDqFY1tc3uzo4dO8jPz6d9+/YVuoY+wS1SjQ4ePEjLli3rfKKQmsXMaNmyZaV6tEoWItVMiUKSobL/7pQsRGqBP7z0QbJDkHpOyUKkiv3hpQ9oN2n+US+AdpPmk//FAVbl72JV/i627Sn/kMCDS9ZXSWw5OTksWrToqLIHHniAcePGlXleamoqAFu2bGHEiBEl1snOzg79YOwDDzzA/v374/sXXnghu3btKk/oJXrjjTdo3749Xbt2pWvXrqSmpnL66afTtWtXrrjiimO61pEjR5gyZUqZdd555x3MjJdffrnCMdda7l4nXj169PDaaOnSpckOodrVxza3vfX/ubv7mjVrKnV+Zf3pT3/yMWPGHFXWq1cv//vf/17meSeccELotfv37+9vv/12fH/Pnj3fqNO2bVv//PPPyxltuDvvvNPnzJlTagzH4tChQ96sWbMy69x0003et29fv+qqq0o8XlKbKxpLFEr690fs7tTQ37HqWYjUIyNGjGD+/Pl89dVXAGzatIktW7bQr18/CgoKGDBgAN27d+fMM8/khRde+Mb5mzZtonPnzgAcOHCAkSNH0rFjR4YOHcqBAwfi9caNG0f//v3p1KkTv/rVrwB46KGH2LJlCzk5OeTk5ACxZXq2b98OwP3330/nzp3p3LkzDzzwQPz9OnbsyDXXXEOnTp0YOHDgUe+zZMkSfvjDH5ba3sLCQm666SZ69uxJZmYmjz32GACbN2+mb9++dO3alc6dO/Paa68xadIk9u7dW2qv5MiRIzz77LM8+eSTLFy4MP49BHj88cfJzMzknHPO4corrwTg008/ZciQIWRmZtKlSxfefPNNNmzYQNeuXePnTZkyhd/+9rcA9O3blxtvvJGsrCymTp3KCy+8QK9evejWrRsDBw7ks88+A2J3XP3kJz8hMzOTzMxM5s6dy/Tp05k4cWL8uo888gg333xzqd+Xikj2ch8i9VbR8FRV19805d9KPXbSSSfRs2dPFi5cyJAhQ5g1axaXXHIJZkaTJk14/vnnOfHEE9m+fTtnn302F110UakTo4888gjf+ta3WLt2LatWraJ79+7xY3fffTeNGjXiW9/6FgMGDGDVqlVcf/313H///SxdupRWrVodda0VK1bw+OOP8+abb+Lu9OrVi/79+9OiRQvWr19Pbm4uf/7zn7nkkkt49tlnufzyy9m+fTuNGjWiWbNmpbZ3+vTpnHzyybz11lt8+eWXnH322QwcOJDc3Fx+9KMfceutt3L48GEOHDhAz549eeyxx3jvvfdKvNby5cs5/fTTOfXUU+nbt2/8e7hy5UruvfdeXnvtNRo1asShQ4cAGD9+POeffz4TJkygsLCQ/fv3x3/hl+bw4cPxobwvvvgi/v1/9NFH+f3vf8+9997LXXfdxbe//W1WrVqFu7Nr1y5SUlLo1q0bU6ZMoWHDhjz++OM8+eSTZb7XsVKyEKlnRo0axaxZs+LJ4i9/+QsQG5K+/fbbWbZsGQ0aNGDz5s1s27aN1q1bl3idZcuWcf311wPE/8otMnv2bB599FGOHDnC1q1bWbNmzVHHi3v11VcZOnRofMXWYcOGsXz5ci666KL4nARAjx492LRpEwCLFy9m4MCBZbZ18eLFrF27llmzZgGwe/du1q9fz1lnncW1117LwYMH+fGPf0yXLl0oLCws81q5ubmMHDkSgJEjR5Kbm8uQIUN45ZVXuPTSSznppJPYu3cvJ510EhBb+qPofRs2bMiJJ54YmiwuvfTS+PbHH3/MJZdcwqeffsqXX37JaaedBsDLL7/M3LmxBbrNjBYtWgBw7rnnsnDhQk499VRSUlLo2LFjme91rJQsRKrBkO9/81OzZfUAims3af4x1S8zliFDuPHGG/nHP/7B/v376dGjBwBPPfUUn3/+OStWrKBRo0a0a9euQvfl/+///i/33Xcfr7zyCt/73vcYM2ZMpe7vP+644+LbKSkp8WGohQsXctNNN5V2GhBLgA8//DADBgz4xrG8vDzmz5/PFVdcwS233HLUL+riDh06xHPPPcf8+fP59a9/zZEjR9i1axf79u0r8/2L98oaNmzIkSNH4vsHDx6kYcOvfw0nLm8+fvx4br/9di688EJefvnl0Mn3q6++mvvvv5927drFh8KqkuYsRKrB0A6Nkx1CXGpqKjk5OVx11VWMGjUqXr57925OPvlkGjVqxNKlS/noo7JXrj733HN5+umnAfjnP//JqlWrANizZw8nnHACzZo1Y9u2bSxcuDB+TtOmTdm7d+83rtWvXz/mzp3L/v372bdvH88//zz9+vUr9b3dnVWrVh01/l+SQYMG8fDDD8d7DevWrePAgQN89NFHtG7dmrFjx3LllVfy7rvvxn9pl9TDeOmllzjrrLP45JNP2LRpEx9//DE/+tGPeOGFFzjvvPN45pln2LlzJ0D8a05ODo8++igQG17as2cPrVu3ZsuWLXzxxRccPHiQ+fNLH1rcvXs3p5xyCu5+1JDS+eefz7Rp0+Lfhy+++AKAPn368OGHH/K3v/2tzMRXUUoWIvXQqFGjWLly5VHJYvTo0bzzzjuceeaZzJw5kzPOOKPMa4wbN46CggI6duzInXfeGe+hdOnShW7dutGjRw8uu+wy+vTpEz9n7NixDB48OD7BXaR79+6MGTOGnj170qtXL66++mq6detW6nuvWLGCbt26hX7Q7Nprr6VDhw7xiexx48ZRWFjIkiVL4nE+99xzXHfddQD89Kc/JTMz8xsT3Lm5uQwdOvSosuHDh5Obm0uXLl245ZZbOPfcc+nTp098Ynnq1KksWrSIM888k6ysLP71r3/RpEkTbr/9drKyshg4cCAZGcUfHvq1u+66i6FDh3LWWWeRlpYWL//Vr37Ftm3b6Ny5M127dmX58q8X4x4xYgTnnntumfM4FVaeW6Zqw0u3ztYe9bnNyb51tjpV1W2kJfnNb37jubm5kV2/oqJsc3kMGjTI8/LySj1emVtnNWchUgvcMKBDskOoUe64445kh1Cj7Nixg7PPPpusrCz69+8fyXsoWYjUAjeef1qyQ5AarGXLlqxfXzWf8i+N5ixERCSUkoWIiIRSshARkVBKFiIiEkrJQqQme/UPsHdblV1ux44d8eW8W7duzSmnnBLfT1wYryxXXnkl69atK7POtGnTeOaZZ6oiZAC2bdtGw4YN4wsBSvXT3VAiNVnBZ/A/D8Lg/6iSy7Vs2TK+UN5dd91FamrqUauVwtefvWrQoOS/JR9//PHQ9xk/fnyJn9SuqNmzZ9O7d29yc3O5+uqrq+y6xRUWFh61/IZ8TT0LkZqszw2w8ukq7V2UZMOGDWRkZDB69Gg6derE1q1bGTt2LFlZWXTq1InJkyfH6/bt25f33nuPwsJCmjdvzqRJk+jSpQu9e/eOL5R3xx13xJek6Nu3L5MmTaJnz56cfvrpvPbaawDs27eP4cOHk5GRwYgRI8jKyip1xdfc3FweeOABNm7cyNatW+Pl8+fPp3v37nTp0iW+qGBJS3gXxVpk1qxZ8aRz+eWXM27cOHr27Mntt9/OG2+8Qe/evenWrRt9+vSJ35JaWFjIjTfeSOfOncnMzOThhx9m8eLFRz0MavHixVx88cWV/nnUREqhIsly1zEsyfD7Y/icxV27jz0W4F//+hczZ84kKysLiD1r4aSTTqKwsJCcnBxGjBjxjeUpdu/eTf/+/ZkyZQo33XQTM2bMYNKkSd+4trvz1ltvMW/ePCZPnsyLL77IH//4R1q3bs2zzz7LypUrj1riPNGmTZvYuXMnPXr04OKLL2b27NnccMMNfPrpp4wbN47ly5fTtm3b+JpMJS3hHWbr1q288cYbNGjQgN27d7N8+XIaNmzIiy++yB133MEzzzzDI488wpYtW1i5ciUpKSns3LmT5s2bM2HCBHbs2EHLli156qmnuOaaa471W18rqGchIgB8//vfjycKiP013717d7p3787atWtZs2bNN845/vjjueCCC4Cjlw8vbtiwYd+o8+qrr8aX/O7SpQudOnUq8dxZs2bFF8YrWhoc4PXXXycnJ4e2bdsCxJcGf/nllxk/fjxw9BLeZbn44ovjw267du1i+PDhdO7cmYkTJ7J69er4dX/2s5+RkpISf78GDRowevRonn76aXbu3Mm7774bumx6baWehUiylKcH8NU+mJ4DfX8BXS+LNJzE5bHXr1/Pgw8+yFtvvUXz5s25/PLLS1xmvHHjr1fTTUlJKfWZEEXLjJdVpzS5ubls3749vvLqli1b2Lhx4zFdo0GDBsSWQYop3pbEtv/yl79k0KBB/PznP2fDhg0MHjy4zGtfddVVDB8+HIgtLliUTOqaSHsWZjbYzNaZ2QYz+0bf1MzamtkSM1tlZnlmlh6U55jZewmvg2b24yhjFamR5k+E9KzIE0Vxe/bsoWnTppx44ols3bqVRYsWVfl79OnTh9mzZwPw/vvvl9hzWbNmDYWFhWzevJlNmzaxadMmbr75ZmbNmsU555xz1FLqRcNQJS3h3aBBg/hT944cOcLzzz9falxFS4MDPPHEE/Hy888/n0cffZTDhw8f9X5t2rShVatWTJkyhcsuq96fU3WKLFmYWQowDbgAyABGmVnx9XjvA2a6eyYwGbgHwN2XuntXd+8KnAfsBxZHFatIjfTuU7DlH3Dh76r9rbt3705GRgZnnHEGV1xxxVHLjFeV6667js2bN5ORkcGvf/1rMjIyvrG0dllLg6elpfHII48wZMgQunTpwujRo4HSl/C+9957GTRoEOeccw7p6emlxnXrrbdy8803071796N6I9deey2tW7eOP1O7KNEBXHbZZbRv354OHerwgo/lWZq2Ii+gN7AoYf824LZidVYDbYJtA/aUcJ2xwFNh76clymuP+tzmY1qifPkf3LdVbEnzmiBsue5Dhw75gQMH3N39gw8+8Hbt2vmhQ4eqI7Qqd+211/oTTzyR9CXKw9TUJcpPAT5J2M8HehWrsxIYBjwIDAWamllLd9+RUGckcH+EcYrUTH1/kewIIlVQUMCAAQMoLCzE3fnTn/5UKz/j0LVrV1q0aMFDDz3El19+mexwImOe0M2q0gubjQAGu/vVwf6/A73cfUJCne8CU4H2wDJgONDZ3XcFx78DrAK+6+6HSniPscR6HqSlpfUoejh6bVJQUEBqamqyw6hW9bnNzZo14wc/+EGyw6kWhw8frrOTvaWp6W3esGEDu3cffWNFTk7OCnfPKuWUuCjT+GagTcJ+elAW5+5biPUsMLNUYHhRoghcAjxfUqIIzp8OTAfIysry7OzsKgu+uuTl5VEb466M+tzmtWvXkpqaGvo40Lpg7969NG3aNNlhVKua3GZ3p0mTJmU+rrYsUd4N9TbQwczam1ljYsNJ8xIrmFkrMyuK4TZgRrFrjAJyI4xRpFo1adKEHTt2EFWPXqQk7s6OHTto0qRJha8RWc/C3QvNbAKwCEgBZrj7ajObTGxCZR6QDdxjZk5sGGp80flm1o5Yz+TvUcUoUt3S09PJz8/n888/T3YokTt48GClfjnVRjW5zU2aNCnzLrAwkc4mufsCYEGxsjsTtucAc0o5dxOxSXKROqNRo0a0b98+2WFUi7y8vAoPedRWdbnNWu5DRERCKVmIiEgoJQsREQmlZCEiIqGULEREJJSShYiIhFKyEBGRUEoWIiISSslCRERCKVmIiEgoJQsREQmlZCEiIqGULEREJJSShYiIhFKyEBGRUEoWIiISSslCRERCKVmIiEgoJQsREQmlZCEiIqGULEREJJSShYiIhFKyEBGRUEoWIiISSslCRERCRZoszGywma0zsw1mNqmE423NbImZrTKzPDNLTzj2PTNbbGZrzWyNmbWLMlYRESldZMnCzFKAacAFQAYwyswyilW7D5jp7pnAZOCehGMzgd+5e0egJ/BZVLGKiEjZouxZ9AQ2uPtGd/8KmAUMKVYnA3gl2F5adDxIKg3d/SUAdy9w9/0RxioiImWIMlmcAnySsJ8flCVaCQwLtocCTc2sJXAasMvMnjOzd83sd0FPRUREkqBhkt9/IjDVzMYAy4DNwGFicfUDugEfA88AY4C/JJ5sZmOBsQBpaWnk5eVVU9hVp6CgoFbGXRlqc/2gNtctUSaLzUCbhP30oCzO3bcQ9CzMLBUY7u67zCwfeM/dNwbH5gJnUyxZuPt0YDpAVlaWZ2dnR9OSCOXl5VEb464Mtbl+UJvrliiHod4GOphZezNrDIwE5iVWMLNWZlYUw23AjIRzm5vZt4P984A1EcYqIiJliCxZuHshMAFYBKwFZrv7ajObbGYXBdWygXVm9gGQBtwdnHuY2BDVEjN7HzDgz1HFKiIiZYt0zsLdFwALipXdmbA9B5hTyrkvAZlRxiciIuWjT3CLiEgoJQsREQmlZCEiIqGULEREJJSShYiIhFKyEBGRUEoWIiISSslCRERCKVmIiEgoJQsREQmlZCEiIqGULEREJJSShYiIhFKyEBGRUKHJwsyuM7MW1RGMiIjUTOXpWaQBb5vZbDMbbGYWdVAiIlKzhCYLd78D6EDs+ddjgPVm9h9m9v2IYxMRkRqiXHMW7u7Ap8GrEGgBzDGz/4wwNhERqSFCH6tqZjcAVwDbgceAm939kJk1ANYDt0QbooiIJFt5nsF9EjDM3T9KLHT3I2b2f6IJS0REapLyDEMtBHYW7ZjZiWbWC8Dd10YVmIiI1BzlSRaPAAUJ+wVBmYiI1BPlSRYWTHADseEnyjd8JSIidUR5ksVGM7vezBoFrxuAjVEHJiIiNUd5ksXPgHOAzUA+0AsYG2VQIiJSs4QOJ7n7Z8DIilzczAYDDwIpwGPuPqXY8bbADODbxCbRL3f3/ODYYeD9oOrH7n5RRWIQEZHKK8/nLJoAPwU6AU2Kyt39qpDzUoBpwPnEeiRvm9k8d1+TUO0+YKa7P2lm5wH3AP8eHDvg7l2PpTEiIhKN8gxD/RVoDQwC/g6kA3vLcV5PYIO7b3T3r4BZwJBidTKAV4LtpSUcFxGRGsASbnQquYLZu+7ezcxWuXummTUClrv72SHnjQAGu/vVwf6/A73cfUJCnaeBN939QTMbBjwLtHL3HWZWCLxHbHmRKe4+t4T3GEswf5KWltZj1qxZx9D0mqGgoIDU1NRkh1Gt1Ob6QW2uHXJycla4e1ZYvfLcAnso+LrLzDoTWx/q5MoEl2AiMNXMxgDLiE2iHw6OtXX3zWZ2KvCKmb3v7h8mnuzu04HpAFlZWZ6dnV1FYVWfvLw8amPclaE21w9qc91SnmQxPXiexR3APCAV+L/lOG8z0CZhPz0oi3P3LcAwADNLBYa7+67g2Obg60YzywO6AUclCxERqR5lJotgscA97v4Fsb/8Tz2Ga78NdDCz9sSSxEjgsmLXbwXsDD7odxuxO6MIktN+d/8yqNMH0Aq3IiJJUuYEd/BLvEKryrp7ITABWASsBWa7+2ozm2xmRbfBZgPrzOwDYg9Zujso7wi8Y2YriU18Tyl2F5WIiFSj8gxDvWxmE4FngH1Fhe6+s/RT4nUWAAuKld2ZsD0HmFPCea8BZ5YjNhERqQblSRaXBl/HJ5Q5xzYkJSIitVh5PsHdvjoCERGRmqs8n+C+oqRyd59Z9eGIiEhNVJ5hqLMStpsAA4B/AEoWIiL1RHmGoa5L3Dez5sSW7hARkXqiPGtDFbcP0DyGiEg9Up45i/8mdvcTxJJLBjA7yqBERKRmKc+cxX0J24XAR0XPnBARkfqhPMniY2Crux8EMLPjzaydu2+KNDIREakxyjNn8TfgSML+4aBMRETqifIki4bBw4sACLYbRxeSiIjUNOVJFp8nLPyHmQ0BtkcXkoiI1DTlmbP4GfCUmU0N9vOBEj/VLSIidVN5PpT3IXB28HAi3L0g8qhERKRGCR2GMrP/MLPm7l7g7gVm1sLMflsdwYmISM1QnjmLC4oedQoQPDXvwuhCEhGRmqY8ySLFzI4r2jGz44HjyqgvIiJ1THkmuJ8ClpjZ44ABY4AnowxKRERqlvJMcN8bPAv7h8TWiFoEtI06MBERqTnKu+rsNmKJ4mLgPGBtZBGJiEiNU2rPwsxOA0YFr+3AM4C5e041xSYiIjVEWcNQ/wKWA//H3TcAmNmN1RKViIjUKGUNQw0DtgJLzezPZjaA2AS3iIjUM6UmC3ef6+4jgTOApcAvgJPN7BEzG1hdAYqISPKFTnC7+z53f9rdfwSkA+8Ct5bn4mY22MzWmdkGM5tUwvG2ZrbEzFaZWZ6ZpRc7fqKZ5SesSyUiIklwTM/gdvcv3H26uw8Iq2tmKcA04AJij2IdZWYZxardB8x090xgMnBPseO/AZYdS4wiIlL1jilZHKOewAZ33xg8A2MWMKRYnQzglWB7aeJxM+sBpAGLI4xRRETKIcpkcQrwScJ+flCWaCWxiXSAoUBTM2tpZg2A3wMTI4xPRETKqTzLfURpIjDVzMYQG27aTOyxrT8HFrh7vlnpN2CZ2VhgLEBaWhp5eXlRx1vlCgoKamXclaE21w9qc90SZbLYDLRJ2E8PyuLcfQtBzyJ4XsZwd99lZr2Bfmb2cyAVaGxmBe4+qdj504HpAFlZWZ6dnR1VWyKTl5dHbYy7MtTm+kFtrluiTBZvAx3MrD2xJDESuCyxgpm1Ana6+xHgNmAGgLuPTqgzBsgqnihERKT6RDZn4e6FwARiCw+uBWa7+2ozm5zwTO9sYJ2ZfUBsMvvuqOIREZGKi3TOwt0XAAuKld2ZsD0HmBNyjSeAJyIIT0REyinKu6FERKSOULIQEZFQShYiIhJKyUJEREIpWYiISCglCxERCaVkISIioZQsREQklJKFiIiEUrIQEZFQShYiIhJKyUJEREIpWYiISCglCxERCaVkISIioZQsREQklJKFiIiEUrIQEZFQShYiIhJKyUJEREIpWYiISCglCxERCaVkISIioZQsREQklJKFiIiEijRZmNlgM1tnZhvMbFIJx9ua2RIzW2VmeWaWnlD+DzN7z8xWm9nPooxTRETKFlmyMLMUYBpwAZABjDKzjGLV7gNmunsmMBm4JyjfCvR2965AL2CSmX03qlhFRKRsUfYsegIb3H2ju38FzAKGFKuTAbwSbC8tOu7uX7n7l0H5cRHHKSIiIRpGeO1TgE8S9vOJ9RISrQSGAQ8CQ4GmZtbS3XeYWRtgPvAD4GZ331L8DcxsLDAWIC0tjby8vCpvRNQKCgpqZdyVoTbXD2pz3RJlsiiPicBUMxsDLAM2A4cB3P0TIDMYfpprZnPcfVviye4+HZgOkJWV5dnZ2dUYetXIy8ujNsZdGWpz/aA21y1RDu9sBtok7KcHZXHuvsXdh7l7N+CXQdmu4nWAfwL9IoxVRETKEGWyeBvoYGbtzawxMBKYl1jBzFqZWVEMtwEzgvJ0Mzs+2G4B9AXWRRiriIiUIbJk4e6FwARgEbAWmO3uq81sspldFFTLBtaZ2QdAGnB3UN4ReNPMVgJ/B+5z9/ejilVERMoW6ZyFuy8AFhQruzNhew4wp4TzXgIyo4xNRETKT7ekiohIKCULEREJpWQhIiKhlCxERCSUkoWIiIRSshARkVBKFiIiEkrJQkREQilZiIhIKCULEREJpWQhIiKhlCxERCSUkoWIiIRSshARkVBKFiIiEkrJQkREQilZiIhIKCULEREJpWQhIiKhlCxERCSUkoWIiIRSshARkVBKFiIiEkrJQkREQkWaLMxssJmtM7MNZjaphONtzWyJma0yszwzSw/Ku5rZ62a2Ojh2aZRxiohI2SJLFmaWAkwDLgAygFFmllGs2n3ATHfPBCYD9wTl+4Er3L0TMBh4wMyaRxWriIiULcqeRU9gg7tvdPevgFnAkGJ1MoBXgu2lRcfd/QN3Xx9sbwE+A74dYawiIlKGKJPFKcAnCfv5QVmilcCwYHv3AXkSAAAHvElEQVQo0NTMWiZWMLOeQGPgw4jiFBGREA2T/P4TgalmNgZYBmwGDhcdNLPvAH8FfuLuR4qfbGZjgbEAaWlp5OXlVUPIVaugoKBWxl0ZanP9oDbXLVEmi81Am4T99KAsLhhiGgZgZqnAcHffFeyfCMwHfunub5T0Bu4+HZgOkJWV5dnZ2VXchOjl5eVRG+OujHrV5lf/AF0uI2/F2vrT5kC9+jkH6nKboxyGehvoYGbtzawxMBKYl1jBzFqZWVEMtwEzgvLGwPPEJr/nRBijSLQKPoP/eTDZUYhUWmTJwt0LgQnAImAtMNvdV5vZZDO7KKiWDawzsw+ANODuoPwS4FxgjJm9F7y6RhWrSGT63AArn6bxl18kOxKRSol0zsLdFwALipXdmbA9B/hGz8Hd/wv4ryhjE6kWTVtDl1G0+eQ5YvdwiNROyZ7gFqn7ThtEmzeGwF3Nji7vPwlybktOTCLHSMlCJEpf7YMFt7D2jBvoOHJysqMRqTCtDSUSpfkTIT2Lba3PS3YkIpWiZCESlXefgi3/gAt/l+xIRCpNyUIkKvs+h4ufgMYnJDsSkUrTnIVIVPr+ItkRiFQZ9SxERCSUkoWIiIRSshARkVBKFiIiEsrcPdkxVAkz+xz4KNlxVEArYHuyg6hmanP9oDbXDm3dPfThcnUmWdRWZvaOu2clO47qpDbXD2pz3aJhKBERCaVkISIioZQskm96sgNIArW5flCb6xDNWYiISCj1LEREJJSSRYTMbLCZrTOzDWY2qYTjbc1siZmtMrM8M0tPOPY9M1tsZmvNbI2ZtavO2Cuqkm3+TzNbHbT5ITOz6o3+2JnZDDP7zMz+WcpxC9qyIWhz94RjPzGz9cHrJ9UXdeVUtM1m1tXMXg9+xqvM7NLqjbziKvNzDo6faGb5Zja1eiKOgLvrFcELSAE+BE4FGgMrgYxidf4G/CTYPg/4a8KxPOD8YDsV+Fay2xRlm4FzgP8JrpECvA5kJ7tN5WjzuUB34J+lHL8QWAgYcDbwZlB+ErAx+Noi2G6R7PZE3ObTgA7B9neBrUDzZLcnyjYnHH8QeBqYmuy2VPSlnkV0egIb3H2ju38FzAKGFKuTAbwSbC8tOm5mGUBDd38JwN0L3H1/9YRdKRVuM+BAE2JJ5jigEbAt8ogryd2XATvLqDIEmOkxbwDNzew7wCDgJXff6e5fAC8Bg6OPuPIq2mZ3/8Dd1wfX2AJ8BoR+GKwmqMTPGTPrAaQBi6OPNDpKFtE5BfgkYT8/KEu0EhgWbA8FmppZS2J/ge0ys+fM7F0z+52ZpUQeceVVuM3u/jqx5LE1eC1y97URx1sdSvuelOd7VVuFts3MehL7w+DDaowrSiW22cwaAL8HJiYlqiqkZJFcE4H+ZvYu0B/YDBwm9pyRfsHxs4gN64xJUoxVrcQ2m9kPgI5AOrH/eOeZWb/khSlRCf7i/itwpbsfSXY8Efs5sMDd85MdSGXp4UfR2Qy0SdhPD8rigq74MAAzSwWGu/suM8sH3nP3jcGxucTGQf9SHYFXQmXafA3whrsXBMcWAr2B5dUReIRK+55sBrKLledVW1TRKvXfgZmdCMwHfhkM19QVpbW5N9DPzH5ObO6xsZkVuPs3bv6o6dSziM7bQAcza29mjYGRwLzECmbWKuimAtwGzEg4t7mZFY3nngesqYaYK6sybf6YWI+joZk1ItbrqAvDUPOAK4K7Zc4Gdrv7VmARMNDMWphZC2BgUFYXlNjm4N/E88TG9uckN8QqV2Kb3X20u3/P3dsR61XPrI2JAtSziIy7F5rZBGK/AFKAGe6+2swmA++4+zxif1neY2YOLAPGB+ceNrOJwJLg9tEVwJ+T0Y5jUZk2A3OIJcX3iU12v+ju/13dbThWZpZLrE2tgh7hr4hNzuPujwILiN0pswHYD1wZHNtpZr8hlmABJrt7WROoNUZF2wxcQuyuopZmNiYoG+Pu71Vb8BVUiTbXGfoEt4iIhNIwlIiIhFKyEBGRUEoWIiISSslCRERCKVmIiEgoJQuREGZ22MzeS3hV2X3yZtautJVMRWoSfc5CJNwBd++a7CBEkkk9C5EKMrNNFnsGx/tm9lawvlVRb+GV4LkGS8zse0F5mpk9b2Yrg9c5waVSzOzPwXMeFpvZ8UH96y32LJNVZjYrSc0UAZQsRMrj+GLDUIkP7dnt7mcCU4EHgrI/Ak+6eybwFPBQUP4Q8Hd370Ls2Qirg/IOwDR37wTsAoYH5ZOAbsF1fhZV40TKQ5/gFgkRLPyWWkL5JuA8d98YrGf1qbu3NLPtwHfc/VBQvtXdW5nZ50C6u3+ZcI12xJ5r0SHYvxVo5O6/NbMXgQJgLjC3aJFFkWRQz0KkcryU7WPxZcJ20RL1AP8GTCPWC3nbzDTHKEmjZCFSOZcmfH092H6N2Iq7AKP5epn1JcA4ADNLMbNmpV00WJm3jbsvBW4FmhFb4lokKfSXiki4480scWXUFxOWmW5hZquI9Q5GBWXXAY+b2c3A53y9AukNwHQz+ymxHsQ4Yk8FLEkK8F9BQjHgIXffVWUtEjlGmrMQqaBgziLL3bcnOxaRqGkYSkREQqlnISIiodSzEBGRUEoWIiISSslCRERCKVmIiEgoJQsREQmlZCEiIqH+P7vRvIjfpyyHAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plotting our accuracy charts\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "history_dict = history.history\n",
    "\n",
    "acc_values = history_dict['acc']\n",
    "val_acc_values = history_dict['val_acc']\n",
    "epochs = range(1, len(loss_values) + 1)\n",
    "\n",
    "line1 = plt.plot(epochs, val_acc_values, label='Validation/Test Accuracy')\n",
    "line2 = plt.plot(epochs, acc_values, label='Training Accuracy')\n",
    "plt.setp(line1, linewidth=2.0, marker = '+', markersize=10.0)\n",
    "plt.setp(line2, linewidth=2.0, marker = '4', markersize=10.0)\n",
    "plt.xlabel('Epochs') \n",
    "plt.ylabel('Accuracy')\n",
    "plt.grid(True)\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Now let's display our Confusion Matrix and Classification Report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.99      0.98       980\n",
      "           1       0.98      1.00      0.99      1135\n",
      "           2       0.99      0.97      0.98      1032\n",
      "           3       0.98      0.98      0.98      1010\n",
      "           4       0.99      0.98      0.98       982\n",
      "           5       0.98      0.98      0.98       892\n",
      "           6       0.99      0.98      0.98       958\n",
      "           7       0.98      0.97      0.98      1028\n",
      "           8       0.98      0.97      0.98       974\n",
      "           9       0.97      0.98      0.97      1009\n",
      "\n",
      "   micro avg       0.98      0.98      0.98     10000\n",
      "   macro avg       0.98      0.98      0.98     10000\n",
      "weighted avg       0.98      0.98      0.98     10000\n",
      "\n",
      "[[ 974    0    0    0    0    1    1    1    3    0]\n",
      " [   0 1130    1    2    0    0    1    0    1    0]\n",
      " [   5    5 1002    4    3    0    0    9    3    1]\n",
      " [   1    0    2  994    0    5    0    5    1    2]\n",
      " [   1    1    0    0  963    0    6    0    2    9]\n",
      " [   3    0    0    6    0  878    2    1    2    0]\n",
      " [   6    3    0    0    2    4  941    0    2    0]\n",
      " [   1    4   10    2    2    0    0  999    1    9]\n",
      " [   7    0    1    2    4    2    2    2  945    9]\n",
      " [   5    6    0    3    3    2    0    3    2  985]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report,confusion_matrix\n",
    "import numpy as np\n",
    "\n",
    "y_pred = model.predict_classes(x_test)\n",
    "\n",
    "print(classification_report(np.argmax(y_test,axis=1), y_pred))\n",
    "print(confusion_matrix(np.argmax(y_test,axis=1), y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Displaying our misclassified data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Indices of misclassifed data are: \n",
      "\n",
      "(array([ 247,  259,  290,  320,  321,  445,  449,  582,  659,  684,  691,\n",
      "        717,  720,  740,  844,  846,  924,  939,  947,  965, 1014, 1039,\n",
      "       1062, 1112, 1181, 1182, 1202, 1226, 1232, 1242, 1247, 1260, 1290,\n",
      "       1299, 1319, 1326, 1393, 1402, 1500, 1522, 1530, 1549, 1553, 1621,\n",
      "       1678, 1681, 1686, 1709, 1717, 1754, 1790, 1878, 1901, 2035, 2043,\n",
      "       2044, 2053, 2070, 2098, 2109, 2118, 2129, 2130, 2135, 2182, 2185,\n",
      "       2189, 2272, 2280, 2293, 2369, 2387, 2406, 2454, 2462, 2488, 2597,\n",
      "       2607, 2654, 2771, 2780, 2810, 2896, 2921, 3060, 3073, 3206, 3289,\n",
      "       3316, 3333, 3422, 3475, 3503, 3520, 3558, 3597, 3662, 3726, 3730,\n",
      "       3767, 3778, 3780, 3796, 3808, 3811, 3836, 3850, 3853, 3906, 3941,\n",
      "       3951, 4007, 4075, 4078, 4102, 4176, 4199, 4205, 4224, 4238, 4248,\n",
      "       4256, 4289, 4360, 4500, 4536, 4571, 4601, 4639, 4740, 4751, 4761,\n",
      "       4783, 4807, 4808, 4874, 4876, 4956, 5067, 5278, 5288, 5331, 5600,\n",
      "       5642, 5734, 5887, 5888, 5937, 5955, 5973, 6004, 6035, 6091, 6505,\n",
      "       6555, 6576, 6597, 6651, 6740, 6796, 6847, 7121, 7432, 7434, 7886,\n",
      "       8059, 8091, 8094, 8095, 8408, 8527, 9009, 9015, 9019, 9024, 9280,\n",
      "       9634, 9642, 9664, 9716, 9726, 9729, 9770, 9792, 9811, 9839, 9888,\n",
      "       9905, 9982]),)\n"
     ]
    }
   ],
   "source": [
    "import cv2\n",
    "import numpy as np\n",
    "from keras.datasets import mnist\n",
    "\n",
    "# loads the MNIST dataset\n",
    "(x_train, y_train), (x_test, y_test)  = mnist.load_data()\n",
    "\n",
    "# Use numpy to create an array that stores a value of 1 when a misclassification occurs\n",
    "result = np.absolute(y_test - y_pred)\n",
    "result_indices = np.nonzero(result > 0)\n",
    "\n",
    "#  Display the indices of mislassifications\n",
    "print(\"Indices of misclassifed data are: \\n\\n\" + str(result_indices))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Displaying the misclassifications"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2 \n",
    "#from keras.models import load_model\n",
    "\n",
    "#classifier = load_model('/home/deeplearningcv/DeepLearningCV/Trained Models/mnist_simple_cnn.h5')\n",
    "\n",
    "def draw_test(name, pred, input_im, true_label):\n",
    "    BLACK = [0,0,0]\n",
    "    expanded_image = cv2.copyMakeBorder(input_im, 0, 0, 0, imageL.shape[0]*2 ,cv2.BORDER_CONSTANT,value=BLACK)\n",
    "    expanded_image = cv2.cvtColor(expanded_image, cv2.COLOR_GRAY2BGR)\n",
    "    cv2.putText(expanded_image, str(pred), (152, 70) , cv2.FONT_HERSHEY_COMPLEX_SMALL,4, (0,255,0), 2)\n",
    "    cv2.putText(expanded_image, str(true_label), (250, 70) , cv2.FONT_HERSHEY_COMPLEX_SMALL,4, (0,0,255), 2)\n",
    "    cv2.imshow(name, expanded_image)\n",
    "\n",
    "for i in range(0,10):\n",
    "\n",
    "    input_im = x_test[result_indices[0][i]]\n",
    "    #print(y_test[result_indices[0][i]])\n",
    "    imageL = cv2.resize(input_im, None, fx=4, fy=4, interpolation = cv2.INTER_CUBIC) \n",
    "    input_im = input_im.reshape(1,28,28,1) \n",
    "    \n",
    "    ## Get Prediction\n",
    "    res = str(model.predict_classes(input_im, 1, verbose = 0)[0])\n",
    "    draw_test(\"Prediction\", res, imageL, y_test[result_indices[0][i]]) \n",
    "    cv2.waitKey(0)\n",
    "\n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}