MNIST dataset using keras api.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "61778889",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tensorflow import keras \n",
    "from tensorflow.keras import layers\n",
    "import tensorflow as tf\n",
    "from sklearn.model_selection import train_test_split \n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ee271400",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('train.csv')\n",
    "test = pd.read_csv('test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f9abb821",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       pixel0  pixel1  pixel2  pixel3  pixel4  pixel5  pixel6  pixel7  pixel8  \\\n",
      "0           0       0       0       0       0       0       0       0       0   \n",
      "1           0       0       0       0       0       0       0       0       0   \n",
      "2           0       0       0       0       0       0       0       0       0   \n",
      "3           0       0       0       0       0       0       0       0       0   \n",
      "4           0       0       0       0       0       0       0       0       0   \n",
      "...       ...     ...     ...     ...     ...     ...     ...     ...     ...   \n",
      "41995       0       0       0       0       0       0       0       0       0   \n",
      "41996       0       0       0       0       0       0       0       0       0   \n",
      "41997       0       0       0       0       0       0       0       0       0   \n",
      "41998       0       0       0       0       0       0       0       0       0   \n",
      "41999       0       0       0       0       0       0       0       0       0   \n",
      "\n",
      "       pixel9  ...  pixel774  pixel775  pixel776  pixel777  pixel778  \\\n",
      "0           0  ...         0         0         0         0         0   \n",
      "1           0  ...         0         0         0         0         0   \n",
      "2           0  ...         0         0         0         0         0   \n",
      "3           0  ...         0         0         0         0         0   \n",
      "4           0  ...         0         0         0         0         0   \n",
      "...       ...  ...       ...       ...       ...       ...       ...   \n",
      "41995       0  ...         0         0         0         0         0   \n",
      "41996       0  ...         0         0         0         0         0   \n",
      "41997       0  ...         0         0         0         0         0   \n",
      "41998       0  ...         0         0         0         0         0   \n",
      "41999       0  ...         0         0         0         0         0   \n",
      "\n",
      "       pixel779  pixel780  pixel781  pixel782  pixel783  \n",
      "0             0         0         0         0         0  \n",
      "1             0         0         0         0         0  \n",
      "2             0         0         0         0         0  \n",
      "3             0         0         0         0         0  \n",
      "4             0         0         0         0         0  \n",
      "...         ...       ...       ...       ...       ...  \n",
      "41995         0         0         0         0         0  \n",
      "41996         0         0         0         0         0  \n",
      "41997         0         0         0         0         0  \n",
      "41998         0         0         0         0         0  \n",
      "41999         0         0         0         0         0  \n",
      "\n",
      "[42000 rows x 784 columns]\n"
     ]
    }
   ],
   "source": [
    "y = train['label']\n",
    "train = train.drop(['label'],axis =1)\n",
    "print(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "caf050c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[[0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " ...\n",
      " [0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]], shape=(42000, 784), dtype=int64)\n",
      "tf.Tensor([1 0 1 ... 7 6 9], shape=(42000,), dtype=int64)\n"
     ]
    }
   ],
   "source": [
    "X = tf.convert_to_tensor(train)\n",
    "y = tf.convert_to_tensor(y)\n",
    "Xtest =tf.convert_to_tensor(test)\n",
    "print(X)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7b715cb2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[   0 4132]\n",
      " [   1 4684]\n",
      " [   2 4177]\n",
      " [   3 4351]\n",
      " [   4 4072]\n",
      " [   5 3795]\n",
      " [   6 4137]\n",
      " [   7 4401]\n",
      " [   8 4063]\n",
      " [   9 4188]]\n"
     ]
    }
   ],
   "source": [
    "unique, counts = np.unique(np.array(y), return_counts=True)#understand distribution of y_train so as to choose metric for classification\n",
    "#As the distribution of y_train in this dataset is somewhat uniform we can use accuarcy as evaluation metric \n",
    "print (np.asarray((unique, counts)).T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5de29f4c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[[0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]], shape=(42000, 784), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "X = tf.cast(X,float)\n",
    "shape = tf.shape( X ) # get dynamic tensor shape\n",
    "X = X/255.0\n",
    "print(X)\n",
    "Xtest = tf.cast(Xtest,float)\n",
    "Xtest = X_test/255.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "a7e75bed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8400, 784)\n",
      "(23520, 784)\n",
      "(10080, 784)\n"
     ]
    }
   ],
   "source": [
    "X_train, X_cv, y_train, y_cv = train_test_split(X.numpy(), y.numpy(), test_size=0.20, random_state=42)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.30, random_state=42)\n",
    "X_train, X_cv= tf.constant(X_train), tf.constant(X_cv)\n",
    "y_train, y_cv = tf.constant(y_train), tf.constant(y_cv)\n",
    "X_test, y_test = tf.constant(X_test), tf.constant(y_test)\n",
    "print(X_cv.shape)\n",
    "print(X_train.shape)\n",
    "print(X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "77ebfc21",
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_schedule = keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=1e-2,decay_steps=100000,decay_rate=0.8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "45072b48",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dropout:  0.09\n",
      "Epoch 1/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.4834 - accuracy: 0.8507\n",
      "Epoch 2/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.2131 - accuracy: 0.9323\n",
      "Epoch 3/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.1525 - accuracy: 0.9500\n",
      "Epoch 4/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.1161 - accuracy: 0.9644\n",
      "Epoch 5/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.1053 - accuracy: 0.9650\n",
      "Epoch 6/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.0937 - accuracy: 0.9713\n",
      "Epoch 7/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.0649 - accuracy: 0.9789\n",
      "Epoch 8/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.0559 - accuracy: 0.9814\n",
      "Epoch 9/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.0638 - accuracy: 0.9793\n",
      "Epoch 10/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.0694 - accuracy: 0.9774\n",
      "Dropout:  0.1\n",
      "Epoch 1/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.4929 - accuracy: 0.8485\n",
      "Epoch 2/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.2185 - accuracy: 0.9311\n",
      "Epoch 3/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.1663 - accuracy: 0.9492\n",
      "Epoch 4/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.1296 - accuracy: 0.9579\n",
      "Epoch 5/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.1122 - accuracy: 0.9617\n",
      "Epoch 6/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.0976 - accuracy: 0.9675\n",
      "Epoch 7/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.0782 - accuracy: 0.9751\n",
      "Epoch 8/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.0702 - accuracy: 0.9771\n",
      "Epoch 9/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.0690 - accuracy: 0.9776\n",
      "Epoch 10/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.0712 - accuracy: 0.9760\n",
      "Dropout:  0.15\n",
      "Epoch 1/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.5528 - accuracy: 0.8282\n",
      "Epoch 2/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.2543 - accuracy: 0.9190\n",
      "Epoch 3/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.1731 - accuracy: 0.9455\n",
      "Epoch 4/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.1517 - accuracy: 0.9505\n",
      "Epoch 5/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.1143 - accuracy: 0.9618\n",
      "Epoch 6/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.1193 - accuracy: 0.9617\n",
      "Epoch 7/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.0996 - accuracy: 0.9661\n",
      "Epoch 8/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.1045 - accuracy: 0.9677\n",
      "Epoch 9/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.0728 - accuracy: 0.9756\n",
      "Epoch 10/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.0815 - accuracy: 0.9707\n",
      "Dropout:  0.18\n",
      "Epoch 1/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.5599 - accuracy: 0.8230\n",
      "Epoch 2/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.2875 - accuracy: 0.9096\n",
      "Epoch 3/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.1951 - accuracy: 0.9369\n",
      "Epoch 4/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.1542 - accuracy: 0.9517\n",
      "Epoch 5/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.1330 - accuracy: 0.9563\n",
      "Epoch 6/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.1212 - accuracy: 0.9594\n",
      "Epoch 7/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.1124 - accuracy: 0.9623\n",
      "Epoch 8/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.0951 - accuracy: 0.9679\n",
      "Epoch 9/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.0831 - accuracy: 0.9717\n",
      "Epoch 10/10\n",
      "263/263 [==============================] - 1s 3ms/step - loss: 0.0875 - accuracy: 0.9704\n"
     ]
    }
   ],
   "source": [
    "initializer1 = tf.keras.initializers.GlorotNormal()#Glorot Normal initializer/Xavier initialization works better for sigmoid layers\n",
    "initializer2 = tf.keras.initializers.HeNormal()#He initalizer works better for Relu layers\n",
    "#In general if we add more sigmoid layers to our neural network then its accuracy decreases as number of sigmoid layers increase\n",
    "#Instead use relu layers or tanh activations\n",
    "#Create a deep neural network using keras library \n",
    "# We can use dropout and Batch Normalization also\n",
    "#optimiser adam is best as compared to gradient descent or rms-prop \n",
    "dropouts = [0.09,0.1,0.15,0.18]\n",
    "for d in dropouts:\n",
    "    print(\"Dropout: \",d)\n",
    "    model = tf.keras.Sequential([\n",
    "    keras.layers.InputLayer(784),\n",
    "    layers.Dense(512,activation =\"relu\" ,name =\"first_hidden_layer\" , kernel_initializer = initializer2),\n",
    "    layers.BatchNormalization(),\n",
    "    layers.Dropout(d),\n",
    "    layers.Dense(200,activation =\"relu\" ,name =\"second_hidden_layer\" , kernel_initializer = initializer2),\n",
    "    layers.BatchNormalization(),\n",
    "    layers.Dropout(d),\n",
    "    layers.Dense(128,activation =\"relu\" ,name =\"third_hidden_layer\" , kernel_initializer = initializer2),\n",
    "    layers.BatchNormalization(),\n",
    "    layers.Dropout(d),\n",
    "    layers.Dense(10,activation =\"softmax\" ,name =\"output_layer\" , kernel_initializer = initializer1)])\n",
    "    model.compile(\n",
    "    loss=keras.losses.SparseCategoricalCrossentropy(), \n",
    "    optimizer=keras.optimizers.Adam(learning_rate=0.001),\n",
    "    metrics=[\"accuracy\"])\n",
    "    model.fit(X_cv, y_cv, batch_size=32, epochs=10, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "3f86e477",
   "metadata": {},
   "outputs": [],
   "source": [
    "d =0.1 \n",
    "model = tf.keras.Sequential([\n",
    "    keras.layers.InputLayer(784),\n",
    "    layers.Dense(512,activation =\"relu\" ,name =\"first_hidden_layer\" , kernel_initializer = initializer2),\n",
    "    layers.BatchNormalization(),\n",
    "    layers.Dropout(d),\n",
    "    layers.Dense(256,activation =\"relu\" ,name =\"second_hidden_layer\" , kernel_initializer = initializer2),\n",
    "    layers.BatchNormalization(),\n",
    "    layers.Dropout(d),\n",
    "    layers.Dense(64,activation =\"relu\" ,name =\"third_hidden_layer\" , kernel_initializer = initializer2),\n",
    "    layers.BatchNormalization(),\n",
    "    layers.Dropout(d),\n",
    "    layers.Dense(10,activation =\"softmax\" ,name =\"output_layer\" , kernel_initializer = initializer1)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "b4e17848",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_18\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " first_hidden_layer (Dense)  (None, 512)               401920    \n",
      "                                                                 \n",
      " batch_normalization_54 (Bat  (None, 512)              2048      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " dropout_54 (Dropout)        (None, 512)               0         \n",
      "                                                                 \n",
      " second_hidden_layer (Dense)  (None, 256)              131328    \n",
      "                                                                 \n",
      " batch_normalization_55 (Bat  (None, 256)              1024      \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " dropout_55 (Dropout)        (None, 256)               0         \n",
      "                                                                 \n",
      " third_hidden_layer (Dense)  (None, 64)                16448     \n",
      "                                                                 \n",
      " batch_normalization_56 (Bat  (None, 64)               256       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " dropout_56 (Dropout)        (None, 64)                0         \n",
      "                                                                 \n",
      " output_layer (Dense)        (None, 10)                650       \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 553,674\n",
      "Trainable params: 552,010\n",
      "Non-trainable params: 1,664\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "94a03d2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(\n",
    "    loss=keras.losses.SparseCategoricalCrossentropy(), \n",
    "    optimizer=keras.optimizers.Adam(learning_rate=lr_schedule),\n",
    "    metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "810bd1ff",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.3539 - accuracy: 0.8899\n",
      "Epoch 2/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.2386 - accuracy: 0.9257\n",
      "Epoch 3/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.1981 - accuracy: 0.9376\n",
      "Epoch 4/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.1784 - accuracy: 0.9448\n",
      "Epoch 5/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.1636 - accuracy: 0.9480\n",
      "Epoch 6/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.1487 - accuracy: 0.9539\n",
      "Epoch 7/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.1425 - accuracy: 0.9558\n",
      "Epoch 8/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.1334 - accuracy: 0.9586\n",
      "Epoch 9/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.1245 - accuracy: 0.9609\n",
      "Epoch 10/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.1201 - accuracy: 0.9612\n",
      "Epoch 11/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.1109 - accuracy: 0.9653\n",
      "Epoch 12/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.1139 - accuracy: 0.9641\n",
      "Epoch 13/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.1056 - accuracy: 0.9665\n",
      "Epoch 14/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.1026 - accuracy: 0.9678\n",
      "Epoch 15/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.1044 - accuracy: 0.9661\n",
      "Epoch 16/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0974 - accuracy: 0.9692\n",
      "Epoch 17/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0917 - accuracy: 0.9706\n",
      "Epoch 18/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0908 - accuracy: 0.9714\n",
      "Epoch 19/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0886 - accuracy: 0.9718\n",
      "Epoch 20/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0851 - accuracy: 0.9722\n",
      "Epoch 21/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0844 - accuracy: 0.9724\n",
      "Epoch 22/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0817 - accuracy: 0.9728\n",
      "Epoch 23/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0842 - accuracy: 0.9726\n",
      "Epoch 24/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0789 - accuracy: 0.9750\n",
      "Epoch 25/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0761 - accuracy: 0.9761\n",
      "Epoch 26/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0717 - accuracy: 0.9774\n",
      "Epoch 27/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0719 - accuracy: 0.9766\n",
      "Epoch 28/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0728 - accuracy: 0.9766\n",
      "Epoch 29/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0701 - accuracy: 0.9777\n",
      "Epoch 30/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0669 - accuracy: 0.9780\n",
      "Epoch 31/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0675 - accuracy: 0.9786\n",
      "Epoch 32/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0672 - accuracy: 0.9779\n",
      "Epoch 33/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0649 - accuracy: 0.9790\n",
      "Epoch 34/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0652 - accuracy: 0.9790\n",
      "Epoch 35/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0643 - accuracy: 0.9787\n",
      "Epoch 36/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0645 - accuracy: 0.9787\n",
      "Epoch 37/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0630 - accuracy: 0.9800\n",
      "Epoch 38/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0619 - accuracy: 0.9793\n",
      "Epoch 39/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0585 - accuracy: 0.9805\n",
      "Epoch 40/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0593 - accuracy: 0.9806\n",
      "Epoch 41/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0592 - accuracy: 0.9806\n",
      "Epoch 42/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0585 - accuracy: 0.9809\n",
      "Epoch 43/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0660 - accuracy: 0.9785\n",
      "Epoch 44/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0601 - accuracy: 0.9804\n",
      "Epoch 45/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0581 - accuracy: 0.9810\n",
      "Epoch 46/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0576 - accuracy: 0.9813\n",
      "Epoch 47/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0570 - accuracy: 0.9813\n",
      "Epoch 48/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0575 - accuracy: 0.9811\n",
      "Epoch 49/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0535 - accuracy: 0.9822\n",
      "Epoch 50/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0545 - accuracy: 0.9817\n",
      "Epoch 51/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0531 - accuracy: 0.9827\n",
      "Epoch 52/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0548 - accuracy: 0.9822\n",
      "Epoch 53/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0549 - accuracy: 0.9820\n",
      "Epoch 54/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0509 - accuracy: 0.9835\n",
      "Epoch 55/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0560 - accuracy: 0.9814\n",
      "Epoch 56/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0516 - accuracy: 0.9827\n",
      "Epoch 57/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0515 - accuracy: 0.9840\n",
      "Epoch 58/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0508 - accuracy: 0.9836\n",
      "Epoch 59/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0519 - accuracy: 0.9833\n",
      "Epoch 60/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0500 - accuracy: 0.9834\n",
      "Epoch 61/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0482 - accuracy: 0.9838\n",
      "Epoch 62/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0483 - accuracy: 0.9840\n",
      "Epoch 63/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0485 - accuracy: 0.9839\n",
      "Epoch 64/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0503 - accuracy: 0.9834\n",
      "Epoch 65/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0460 - accuracy: 0.9841\n",
      "Epoch 66/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0465 - accuracy: 0.9846\n",
      "Epoch 67/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0467 - accuracy: 0.9848\n",
      "Epoch 68/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0459 - accuracy: 0.9846\n",
      "Epoch 69/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0447 - accuracy: 0.9856\n",
      "Epoch 70/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0459 - accuracy: 0.9851\n",
      "Epoch 71/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0469 - accuracy: 0.9849\n",
      "Epoch 72/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0453 - accuracy: 0.9850\n",
      "Epoch 73/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0470 - accuracy: 0.9847\n",
      "Epoch 74/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0437 - accuracy: 0.9855\n",
      "Epoch 75/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0431 - accuracy: 0.9860\n",
      "Epoch 76/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0467 - accuracy: 0.9844\n",
      "Epoch 77/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0456 - accuracy: 0.9848\n",
      "Epoch 78/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0417 - accuracy: 0.9858\n",
      "Epoch 79/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0406 - accuracy: 0.9865\n",
      "Epoch 80/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0414 - accuracy: 0.9862\n",
      "Epoch 81/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0432 - accuracy: 0.9855\n",
      "Epoch 82/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0440 - accuracy: 0.9860\n",
      "Epoch 83/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0427 - accuracy: 0.9861\n",
      "Epoch 84/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0396 - accuracy: 0.9870\n",
      "Epoch 85/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0430 - accuracy: 0.9852\n",
      "Epoch 86/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0401 - accuracy: 0.9873\n",
      "Epoch 87/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0426 - accuracy: 0.9861\n",
      "Epoch 88/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0411 - accuracy: 0.9866\n",
      "Epoch 89/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0411 - accuracy: 0.9864\n",
      "Epoch 90/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0424 - accuracy: 0.9861\n",
      "Epoch 91/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0403 - accuracy: 0.9868\n",
      "Epoch 92/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0423 - accuracy: 0.9863\n",
      "Epoch 93/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0399 - accuracy: 0.9869\n",
      "Epoch 94/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0410 - accuracy: 0.9862\n",
      "Epoch 95/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0399 - accuracy: 0.9865\n",
      "Epoch 96/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0388 - accuracy: 0.9870\n",
      "Epoch 97/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0375 - accuracy: 0.9877\n",
      "Epoch 98/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0385 - accuracy: 0.9870\n",
      "Epoch 99/100\n",
      "1313/1313 [==============================] - 3s 3ms/step - loss: 0.0379 - accuracy: 0.9879\n",
      "Epoch 100/100\n",
      "1313/1313 [==============================] - 4s 3ms/step - loss: 0.0377 - accuracy: 0.9875\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x267260a9370>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X, y, batch_size=32, epochs=100, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "1bb80071",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "315/315 [==============================] - 0s 1ms/step - loss: 0.0033 - accuracy: 0.9991\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.0032670360524207354, 0.9991071224212646]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(X_test, y_test, batch_size=32, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f06cf12d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}