hyperparam_tests.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2514,
     "status": "ok",
     "timestamp": 1586799530930,
     "user": {
      "displayName": "Steven Su",
      "photoUrl": "",
      "userId": "12872522105664994411"
     },
     "user_tz": 360
    },
    "id": "nJmca26iyZET",
    "outputId": "268370c3-dcdc-45fd-84b0-eaac774382c4"
   },
   "outputs": [],
   "source": [
    "import keras\n",
    "import numpy as np\n",
    "from numpy import load\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from keras.models import Sequential\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras.layers import Dense, Conv2D, MaxPooling2D, Flatten, Dropout, BatchNormalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 101
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6910,
     "status": "ok",
     "timestamp": 1586799538547,
     "user": {
      "displayName": "Steven Su",
      "photoUrl": "",
      "userId": "12872522105664994411"
     },
     "user_tz": 360
    },
    "id": "wZ59xrUVtEgd",
    "outputId": "961312a4-f2a5-4853-b077-734ceac2cb80"
   },
   "outputs": [],
   "source": [
    "#unpack data from source\n",
    "\n",
    "data = np.load('train_and_test.npz')  #what kind of file is this? zipped np.array file\n",
    "\n",
    "X = [data[key] for key in data]  #??? why does this work? \"data\" is not a dictionary data type but a NpzFile type\n",
    "\n",
    "xtrain0 = X[0]\n",
    "ytrain0 = X[1]  #ytrain0 contains values between 0-42; so 43 categories\n",
    "xtest0 = X[2]   #this is test data for final kaggle submission\n",
    "\n",
    "x_tr, x_tst, y_tr, y_tst = train_test_split(xtrain0, ytrain0, test_size = 0.2,shuffle = True)  #results overfit if data is not shuffled!!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nI4NTgHSMKIT"
   },
   "outputs": [],
   "source": [
    "#build model -  this is a function to build the model where activation type and batch\n",
    "#normalization can be specified as parameters so models can be changed without retyping\n",
    "\n",
    "def build_model(activation, batch_normalization = False):\n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(8, (3,3), activation = activation, padding = 'same', name = 'conv1', \n",
    "                    input_shape = (32,32,3))) \n",
    "    if batch_normalization: \n",
    "        model.add(BatchNormalization())\n",
    "    model.add(MaxPooling2D((2,2), name = 'maxpool1')) \n",
    "    model.add(Conv2D(16, (3,3), activation = activation, padding = 'same', name = 'conv2' ))\n",
    "    if batch_normalization: \n",
    "        model.add(BatchNormalization())\n",
    "    model.add(MaxPooling2D((2,2), name = 'maxpool2'))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(64, activation=activation, name='dense_1'))\n",
    "    model.add(Dense(43, activation='softmax', name='output'))  #there are 43 labels\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Optimizing Hyperparameters\n",
    "\n",
    "The remaining of this notebook will be dedicated to testing for the optimal values of our hyperparameters within our model (specifically our learning rate and our momentum). For our own sanity, we will be setting our number of epochs to 20 for our initial tests to limit processing time. Once a proper learning rate has been selected, we will shift our focus to finding the optimal momentum rate. Finally, we will look for the proper number of epochs to run based off our selected learning rate and momentum."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.5, 0.25, 0.125, 0.01, 0.001]\n",
      "[0.99, 0.97, 0.95, 0.93, 0.91, 0.5, 0.25, 0.1, 0]\n"
     ]
    }
   ],
   "source": [
    "learning_rates = []\n",
    "momentums = []\n",
    "\n",
    "rate = 1\n",
    "m = 0.99\n",
    "for i in range(5):\n",
    "    if rate > 0.125:\n",
    "        rate = rate/2\n",
    "    elif rate == 0.125:\n",
    "        rate = 0.01\n",
    "    else:\n",
    "        rate = rate/10\n",
    "        \n",
    "    momentums.append(round(m,2))\n",
    "    m -= 0.02\n",
    "    learning_rates.append(rate)\n",
    "    \n",
    "momentums += [0.5, 0.25, 0.1, 0]\n",
    "\n",
    "test = [0.1,0.25,0.9,0.95,0.99]\n",
    "    \n",
    "print(learning_rates)\n",
    "print(momentums)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "learning rate: 0.5\n",
      "momentum: 0.1\n",
      "Train on 26270 samples, validate on 12939 samples\n",
      "Epoch 1/20\n",
      "26270/26270 [==============================] - 11s 416us/step - loss: 2.7336 - accuracy: 0.2229 - val_loss: 2.2082 - val_accuracy: 0.3273\n",
      "Epoch 2/20\n",
      "26270/26270 [==============================] - 11s 406us/step - loss: 2.0037 - accuracy: 0.3645 - val_loss: 1.7795 - val_accuracy: 0.4207\n",
      "Epoch 3/20\n",
      "26270/26270 [==============================] - 11s 413us/step - loss: 1.7876 - accuracy: 0.4243 - val_loss: 1.6838 - val_accuracy: 0.4667\n",
      "Epoch 4/20\n",
      "26270/26270 [==============================] - 11s 428us/step - loss: 1.6081 - accuracy: 0.4787 - val_loss: 1.7819 - val_accuracy: 0.4538\n",
      "Epoch 5/20\n",
      "26270/26270 [==============================] - 13s 488us/step - loss: 1.6143 - accuracy: 0.4858 - val_loss: 1.4756 - val_accuracy: 0.5313\n",
      "Epoch 6/20\n",
      "26270/26270 [==============================] - 11s 434us/step - loss: 1.4448 - accuracy: 0.5416 - val_loss: 1.3094 - val_accuracy: 0.5834\n",
      "Epoch 7/20\n",
      "26270/26270 [==============================] - 11s 426us/step - loss: 1.6353 - accuracy: 0.4936 - val_loss: 1.3672 - val_accuracy: 0.5572\n",
      "Epoch 8/20\n",
      "26270/26270 [==============================] - 11s 426us/step - loss: 1.2836 - accuracy: 0.5905 - val_loss: 1.0205 - val_accuracy: 0.6616\n",
      "Epoch 9/20\n",
      "26270/26270 [==============================] - 11s 426us/step - loss: 1.2806 - accuracy: 0.5946 - val_loss: 1.0771 - val_accuracy: 0.6537\n",
      "Epoch 10/20\n",
      "26270/26270 [==============================] - 11s 428us/step - loss: 1.3063 - accuracy: 0.5914 - val_loss: 1.1641 - val_accuracy: 0.6302\n",
      "Epoch 11/20\n",
      "26270/26270 [==============================] - 11s 427us/step - loss: 1.2885 - accuracy: 0.6035 - val_loss: 0.8868 - val_accuracy: 0.7083\n",
      "Epoch 12/20\n",
      "26270/26270 [==============================] - 11s 428us/step - loss: 1.0287 - accuracy: 0.6744 - val_loss: 0.8659 - val_accuracy: 0.7188\n",
      "Epoch 13/20\n",
      "26270/26270 [==============================] - 11s 429us/step - loss: 1.0621 - accuracy: 0.6641 - val_loss: 1.1288 - val_accuracy: 0.6359\n",
      "Epoch 14/20\n",
      "26270/26270 [==============================] - 11s 430us/step - loss: 1.0745 - accuracy: 0.6541 - val_loss: 0.7154 - val_accuracy: 0.7654\n",
      "Epoch 15/20\n",
      "26270/26270 [==============================] - 11s 428us/step - loss: 0.8017 - accuracy: 0.7432 - val_loss: 0.6586 - val_accuracy: 0.7788\n",
      "Epoch 16/20\n",
      "26270/26270 [==============================] - 11s 435us/step - loss: 0.6617 - accuracy: 0.7892 - val_loss: 0.6170 - val_accuracy: 0.7926\n",
      "Epoch 17/20\n",
      "26270/26270 [==============================] - 12s 454us/step - loss: 0.7007 - accuracy: 0.7721 - val_loss: 0.5619 - val_accuracy: 0.8171\n",
      "Epoch 18/20\n",
      "26270/26270 [==============================] - 11s 429us/step - loss: 0.8063 - accuracy: 0.7398 - val_loss: 0.6633 - val_accuracy: 0.7831\n",
      "Epoch 19/20\n",
      "26270/26270 [==============================] - 11s 431us/step - loss: 0.5849 - accuracy: 0.8124 - val_loss: 0.7007 - val_accuracy: 0.7763\n",
      "Epoch 20/20\n",
      "26270/26270 [==============================] - 11s 432us/step - loss: 0.5699 - accuracy: 0.8169 - val_loss: 0.5091 - val_accuracy: 0.8341\n",
      "========================================================================================================\n",
      "learning rate: 0.25\n",
      "momentum: 0.25\n",
      "Train on 26270 samples, validate on 12939 samples\n",
      "Epoch 1/20\n",
      "26270/26270 [==============================] - 11s 433us/step - loss: 2.9026 - accuracy: 0.1879 - val_loss: 2.1030 - val_accuracy: 0.3645\n",
      "Epoch 2/20\n",
      "26270/26270 [==============================] - 11s 429us/step - loss: 1.8361 - accuracy: 0.4057 - val_loss: 1.6100 - val_accuracy: 0.4759\n",
      "Epoch 3/20\n",
      "26270/26270 [==============================] - 11s 431us/step - loss: 1.5915 - accuracy: 0.4838 - val_loss: 1.7227 - val_accuracy: 0.4612\n",
      "Epoch 4/20\n",
      "26270/26270 [==============================] - 11s 430us/step - loss: 1.3963 - accuracy: 0.5496 - val_loss: 1.2071 - val_accuracy: 0.6046\n",
      "Epoch 5/20\n",
      "26270/26270 [==============================] - 11s 429us/step - loss: 1.1325 - accuracy: 0.6351 - val_loss: 1.0459 - val_accuracy: 0.6592\n",
      "Epoch 6/20\n",
      "26270/26270 [==============================] - 12s 438us/step - loss: 0.9061 - accuracy: 0.7075 - val_loss: 0.7659 - val_accuracy: 0.7523\n",
      "Epoch 7/20\n",
      "26270/26270 [==============================] - 11s 428us/step - loss: 0.7811 - accuracy: 0.7534 - val_loss: 0.5896 - val_accuracy: 0.8194\n",
      "Epoch 8/20\n",
      "26270/26270 [==============================] - 11s 427us/step - loss: 0.7526 - accuracy: 0.7635 - val_loss: 1.1491 - val_accuracy: 0.6374\n",
      "Epoch 9/20\n",
      "26270/26270 [==============================] - 11s 429us/step - loss: 0.6613 - accuracy: 0.7880 - val_loss: 0.4994 - val_accuracy: 0.8388\n",
      "Epoch 10/20\n",
      "26270/26270 [==============================] - 11s 430us/step - loss: 0.4727 - accuracy: 0.8504 - val_loss: 0.4607 - val_accuracy: 0.8542\n",
      "Epoch 11/20\n",
      "26270/26270 [==============================] - 11s 428us/step - loss: 0.4511 - accuracy: 0.8552 - val_loss: 0.4550 - val_accuracy: 0.8561\n",
      "Epoch 12/20\n",
      "26270/26270 [==============================] - 11s 431us/step - loss: 0.4096 - accuracy: 0.8722 - val_loss: 0.4152 - val_accuracy: 0.8642\n",
      "Epoch 13/20\n",
      "26270/26270 [==============================] - 11s 430us/step - loss: 0.4090 - accuracy: 0.8694 - val_loss: 0.4894 - val_accuracy: 0.8443\n",
      "Epoch 14/20\n",
      "26270/26270 [==============================] - 11s 429us/step - loss: 0.3652 - accuracy: 0.8831 - val_loss: 0.3401 - val_accuracy: 0.8913\n",
      "Epoch 15/20\n",
      "26270/26270 [==============================] - 11s 431us/step - loss: 0.3205 - accuracy: 0.8984 - val_loss: 0.3705 - val_accuracy: 0.8817\n",
      "Epoch 16/20\n",
      "26270/26270 [==============================] - 11s 435us/step - loss: 0.3425 - accuracy: 0.8916 - val_loss: 0.4386 - val_accuracy: 0.8563\n",
      "Epoch 17/20\n",
      "26270/26270 [==============================] - 11s 428us/step - loss: 0.4107 - accuracy: 0.8678 - val_loss: 0.3091 - val_accuracy: 0.8984\n",
      "Epoch 18/20\n",
      "26270/26270 [==============================] - 11s 429us/step - loss: 0.3051 - accuracy: 0.9034 - val_loss: 0.3091 - val_accuracy: 0.8981\n",
      "Epoch 19/20\n",
      "26270/26270 [==============================] - 11s 432us/step - loss: 0.2964 - accuracy: 0.9058 - val_loss: 0.3024 - val_accuracy: 0.9005\n",
      "Epoch 20/20\n",
      "26270/26270 [==============================] - 11s 430us/step - loss: 0.2791 - accuracy: 0.9113 - val_loss: 0.3015 - val_accuracy: 0.9049\n",
      "========================================================================================================\n",
      "learning rate: 0.125\n",
      "momentum: 0.9\n",
      "Train on 26270 samples, validate on 12939 samples\n",
      "Epoch 1/20\n",
      "26270/26270 [==============================] - 11s 437us/step - loss: 3.5129 - accuracy: 0.0536 - val_loss: 3.4961 - val_accuracy: 0.0553\n",
      "Epoch 2/20\n",
      "26270/26270 [==============================] - 12s 439us/step - loss: 3.5054 - accuracy: 0.0532 - val_loss: 3.4971 - val_accuracy: 0.0531\n",
      "Epoch 3/20\n",
      "26270/26270 [==============================] - 11s 436us/step - loss: 3.2514 - accuracy: 0.1125 - val_loss: 2.3633 - val_accuracy: 0.2839\n",
      "Epoch 4/20\n",
      "26270/26270 [==============================] - 13s 479us/step - loss: 2.2057 - accuracy: 0.3231 - val_loss: 2.1487 - val_accuracy: 0.3287\n",
      "Epoch 5/20\n",
      "26270/26270 [==============================] - 12s 456us/step - loss: 2.3451 - accuracy: 0.3033 - val_loss: 2.3992 - val_accuracy: 0.2993\n",
      "Epoch 6/20\n",
      "26270/26270 [==============================] - 11s 437us/step - loss: 2.2728 - accuracy: 0.3216 - val_loss: 2.1114 - val_accuracy: 0.3629\n",
      "Epoch 7/20\n",
      "26270/26270 [==============================] - 12s 443us/step - loss: 2.2025 - accuracy: 0.3385 - val_loss: 3.2336 - val_accuracy: 0.1034\n",
      "Epoch 8/20\n",
      "26270/26270 [==============================] - 11s 433us/step - loss: 2.0104 - accuracy: 0.3884 - val_loss: 1.5938 - val_accuracy: 0.5018\n",
      "Epoch 9/20\n",
      "26270/26270 [==============================] - 11s 434us/step - loss: 1.7592 - accuracy: 0.4557 - val_loss: 1.6208 - val_accuracy: 0.4915\n",
      "Epoch 10/20\n",
      "26270/26270 [==============================] - 11s 435us/step - loss: 1.8688 - accuracy: 0.4314 - val_loss: 1.4034 - val_accuracy: 0.5621\n",
      "Epoch 11/20\n",
      "26270/26270 [==============================] - 11s 436us/step - loss: 1.4747 - accuracy: 0.5391 - val_loss: 1.3505 - val_accuracy: 0.5666\n",
      "Epoch 12/20\n",
      "26270/26270 [==============================] - 12s 442us/step - loss: 1.4399 - accuracy: 0.5440 - val_loss: 2.0167 - val_accuracy: 0.3965\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 13/20\n",
      "26270/26270 [==============================] - 11s 423us/step - loss: 1.9904 - accuracy: 0.4126 - val_loss: 2.9363 - val_accuracy: 0.1915\n",
      "Epoch 14/20\n",
      "26270/26270 [==============================] - 11s 425us/step - loss: 2.5002 - accuracy: 0.2923 - val_loss: 1.7304 - val_accuracy: 0.4792\n",
      "Epoch 15/20\n",
      "26270/26270 [==============================] - 11s 425us/step - loss: 2.0306 - accuracy: 0.4064 - val_loss: 2.1114 - val_accuracy: 0.3777\n",
      "Epoch 16/20\n",
      "26270/26270 [==============================] - 11s 432us/step - loss: 1.5027 - accuracy: 0.5442 - val_loss: 1.1852 - val_accuracy: 0.6237\n",
      "Epoch 17/20\n",
      "26270/26270 [==============================] - 11s 431us/step - loss: 1.2322 - accuracy: 0.6083 - val_loss: 1.1600 - val_accuracy: 0.6333\n",
      "Epoch 18/20\n",
      "26270/26270 [==============================] - 13s 499us/step - loss: 1.5515 - accuracy: 0.5274 - val_loss: 2.1130 - val_accuracy: 0.3697\n",
      "Epoch 19/20\n",
      "26270/26270 [==============================] - 12s 441us/step - loss: 1.8282 - accuracy: 0.4449 - val_loss: 2.5321 - val_accuracy: 0.2751\n",
      "Epoch 20/20\n",
      "26270/26270 [==============================] - 12s 448us/step - loss: 1.7563 - accuracy: 0.4660 - val_loss: 2.3920 - val_accuracy: 0.3040\n",
      "========================================================================================================\n",
      "learning rate: 0.01\n",
      "momentum: 0.95\n",
      "Train on 26270 samples, validate on 12939 samples\n",
      "Epoch 1/20\n",
      "26270/26270 [==============================] - 12s 452us/step - loss: 3.2577 - accuracy: 0.1273 - val_loss: 2.4672 - val_accuracy: 0.3077\n",
      "Epoch 2/20\n",
      "26270/26270 [==============================] - 11s 436us/step - loss: 2.0705 - accuracy: 0.3562 - val_loss: 1.8897 - val_accuracy: 0.4052\n",
      "Epoch 3/20\n",
      "26270/26270 [==============================] - 11s 437us/step - loss: 1.8084 - accuracy: 0.4142 - val_loss: 1.7285 - val_accuracy: 0.4469\n",
      "Epoch 4/20\n",
      "26270/26270 [==============================] - 12s 447us/step - loss: 1.6387 - accuracy: 0.4704 - val_loss: 1.5570 - val_accuracy: 0.5015\n",
      "Epoch 5/20\n",
      "26270/26270 [==============================] - 12s 447us/step - loss: 1.4491 - accuracy: 0.5381 - val_loss: 1.4398 - val_accuracy: 0.5524\n",
      "Epoch 6/20\n",
      "26270/26270 [==============================] - 12s 443us/step - loss: 1.2537 - accuracy: 0.5971 - val_loss: 1.2116 - val_accuracy: 0.6253\n",
      "Epoch 7/20\n",
      "26270/26270 [==============================] - 12s 442us/step - loss: 1.0926 - accuracy: 0.6563 - val_loss: 1.0675 - val_accuracy: 0.6612\n",
      "Epoch 8/20\n",
      "26270/26270 [==============================] - 12s 440us/step - loss: 0.9610 - accuracy: 0.6995 - val_loss: 0.8458 - val_accuracy: 0.7411\n",
      "Epoch 9/20\n",
      "26270/26270 [==============================] - 12s 438us/step - loss: 0.8834 - accuracy: 0.7248 - val_loss: 0.9108 - val_accuracy: 0.7113\n",
      "Epoch 10/20\n",
      "26270/26270 [==============================] - 11s 433us/step - loss: 0.9032 - accuracy: 0.7162 - val_loss: 0.8796 - val_accuracy: 0.7229\n",
      "Epoch 11/20\n",
      "26270/26270 [==============================] - 11s 436us/step - loss: 0.7490 - accuracy: 0.7639 - val_loss: 0.6224 - val_accuracy: 0.8039\n",
      "Epoch 12/20\n",
      "26270/26270 [==============================] - 11s 433us/step - loss: 0.6094 - accuracy: 0.8110 - val_loss: 0.5692 - val_accuracy: 0.8266\n",
      "Epoch 13/20\n",
      "26270/26270 [==============================] - 11s 433us/step - loss: 0.5065 - accuracy: 0.8483 - val_loss: 0.4717 - val_accuracy: 0.8586\n",
      "Epoch 14/20\n",
      "26270/26270 [==============================] - 11s 432us/step - loss: 0.4639 - accuracy: 0.8600 - val_loss: 0.4461 - val_accuracy: 0.8653\n",
      "Epoch 15/20\n",
      "26270/26270 [==============================] - 11s 433us/step - loss: 0.4978 - accuracy: 0.8469 - val_loss: 0.4672 - val_accuracy: 0.8541\n",
      "Epoch 16/20\n",
      "26270/26270 [==============================] - 11s 436us/step - loss: 0.3940 - accuracy: 0.8791 - val_loss: 0.3542 - val_accuracy: 0.8940\n",
      "Epoch 17/20\n",
      "26270/26270 [==============================] - 12s 438us/step - loss: 0.3591 - accuracy: 0.8934 - val_loss: 0.3910 - val_accuracy: 0.8763\n",
      "Epoch 18/20\n",
      "26270/26270 [==============================] - 11s 434us/step - loss: 0.3561 - accuracy: 0.8911 - val_loss: 0.4010 - val_accuracy: 0.8732\n",
      "Epoch 19/20\n",
      "26270/26270 [==============================] - 11s 433us/step - loss: 0.2900 - accuracy: 0.9135 - val_loss: 0.4087 - val_accuracy: 0.8706\n",
      "Epoch 20/20\n",
      "26270/26270 [==============================] - 12s 442us/step - loss: 0.2921 - accuracy: 0.9153 - val_loss: 0.3219 - val_accuracy: 0.9037\n",
      "========================================================================================================\n",
      "learning rate: 0.001\n",
      "momentum: 0.99\n",
      "Train on 26270 samples, validate on 12939 samples\n",
      "Epoch 1/20\n",
      "26270/26270 [==============================] - 12s 458us/step - loss: 3.5140 - accuracy: 0.0596 - val_loss: 3.4542 - val_accuracy: 0.1014\n",
      "Epoch 2/20\n",
      "26270/26270 [==============================] - 11s 429us/step - loss: 3.2349 - accuracy: 0.1257 - val_loss: 2.9368 - val_accuracy: 0.1846\n",
      "Epoch 3/20\n",
      "26270/26270 [==============================] - 11s 431us/step - loss: 2.4945 - accuracy: 0.2821 - val_loss: 2.1941 - val_accuracy: 0.3338\n",
      "Epoch 4/20\n",
      "26270/26270 [==============================] - 11s 432us/step - loss: 2.0093 - accuracy: 0.3591 - val_loss: 1.8956 - val_accuracy: 0.4009\n",
      "Epoch 5/20\n",
      "26270/26270 [==============================] - 11s 435us/step - loss: 1.8709 - accuracy: 0.3913 - val_loss: 1.7894 - val_accuracy: 0.4208\n",
      "Epoch 6/20\n",
      "26270/26270 [==============================] - 11s 433us/step - loss: 1.7686 - accuracy: 0.4192 - val_loss: 1.7567 - val_accuracy: 0.4226\n",
      "Epoch 7/20\n",
      "26270/26270 [==============================] - 11s 438us/step - loss: 1.6840 - accuracy: 0.4445 - val_loss: 1.6861 - val_accuracy: 0.4452\n",
      "Epoch 8/20\n",
      "26270/26270 [==============================] - 11s 432us/step - loss: 1.6013 - accuracy: 0.4743 - val_loss: 1.5618 - val_accuracy: 0.4875\n",
      "Epoch 9/20\n",
      "26270/26270 [==============================] - 11s 433us/step - loss: 1.5542 - accuracy: 0.4937 - val_loss: 1.5286 - val_accuracy: 0.5001\n",
      "Epoch 10/20\n",
      "26270/26270 [==============================] - 11s 430us/step - loss: 1.4435 - accuracy: 0.5335 - val_loss: 1.3873 - val_accuracy: 0.5501\n",
      "Epoch 11/20\n",
      "26270/26270 [==============================] - 11s 425us/step - loss: 1.3300 - accuracy: 0.5797 - val_loss: 1.2979 - val_accuracy: 0.5951\n",
      "Epoch 12/20\n",
      "26270/26270 [==============================] - 11s 433us/step - loss: 1.1633 - accuracy: 0.6392 - val_loss: 1.0860 - val_accuracy: 0.6594\n",
      "Epoch 13/20\n",
      "26270/26270 [==============================] - 11s 433us/step - loss: 0.9979 - accuracy: 0.6968 - val_loss: 0.9786 - val_accuracy: 0.6873\n",
      "Epoch 14/20\n",
      "26270/26270 [==============================] - 11s 434us/step - loss: 0.8634 - accuracy: 0.7443 - val_loss: 0.8249 - val_accuracy: 0.7532\n",
      "Epoch 15/20\n",
      "26270/26270 [==============================] - 11s 433us/step - loss: 0.7550 - accuracy: 0.7780 - val_loss: 0.7620 - val_accuracy: 0.7725\n",
      "Epoch 16/20\n",
      "26270/26270 [==============================] - 11s 433us/step - loss: 0.6494 - accuracy: 0.8126 - val_loss: 0.6045 - val_accuracy: 0.8292\n",
      "Epoch 17/20\n",
      "26270/26270 [==============================] - 11s 437us/step - loss: 0.5711 - accuracy: 0.8362 - val_loss: 0.5378 - val_accuracy: 0.8480\n",
      "Epoch 18/20\n",
      "26270/26270 [==============================] - 12s 442us/step - loss: 0.4924 - accuracy: 0.8610 - val_loss: 0.4679 - val_accuracy: 0.8690\n",
      "Epoch 19/20\n",
      "26270/26270 [==============================] - 12s 448us/step - loss: 0.4705 - accuracy: 0.8674 - val_loss: 0.5856 - val_accuracy: 0.8256\n",
      "Epoch 20/20\n",
      "26270/26270 [==============================] - 11s 437us/step - loss: 0.4403 - accuracy: 0.8772 - val_loss: 0.4286 - val_accuracy: 0.8764\n",
      "========================================================================================================\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(learning_rates)):\n",
    "    print('learning rate:', learning_rates[i])\n",
    "    print('momentum:', test[i])\n",
    "    model = build_model(activation = 'sigmoid', batch_normalization = False)  # try 'relu' too\n",
    "    opt = tf.keras.optimizers.SGD(learning_rate = learning_rates[i], momentum = test[i], nesterov=False)\n",
    "    model.compile(loss = 'categorical_crossentropy', optimizer = opt, metrics = ['accuracy'])\n",
    "    model.fit(x_tr, y_tr, epochs = 20, validation_data= (x_tst, y_tst))\n",
    "    print('========================================================================================================')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The optimal learning rate seems to be around 0.125 (which is what similar to our original value that we were using to begin with). With that found, we will try to find the best matching momentum value and number of epochs. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1804,
     "status": "ok",
     "timestamp": 1586799674783,
     "user": {
      "displayName": "Steven Su",
      "photoUrl": "",
      "userId": "12872522105664994411"
     },
     "user_tz": 360
    },
    "id": "oEVXvN8yf158",
    "outputId": "c0b67229-4cee-4626-a4f4-8e8aa3979b4e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12939/12939 [==============================] - 2s 134us/step\n",
      "Test accuracy: 0.0531\n"
     ]
    }
   ],
   "source": [
    "loss, accuracy = model.evaluate(x_tst, y_tst)\n",
    "print('Test accuracy: %.4f' % (accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "momentum: 0.5\n",
      "Train on 26270 samples, validate on 12939 samples\n",
      "Epoch 1/20\n",
      "26270/26270 [==============================] - 11s 404us/step - loss: 2.8736 - accuracy: 0.2022 - val_loss: 1.9894 - val_accuracy: 0.3722\n",
      "Epoch 2/20\n",
      "26270/26270 [==============================] - 10s 398us/step - loss: 1.9140 - accuracy: 0.3903 - val_loss: 1.7960 - val_accuracy: 0.4192\n",
      "Epoch 3/20\n",
      "26270/26270 [==============================] - 12s 438us/step - loss: 1.6211 - accuracy: 0.4773 - val_loss: 1.4335 - val_accuracy: 0.5325\n",
      "Epoch 4/20\n",
      "26270/26270 [==============================] - 11s 422us/step - loss: 1.2838 - accuracy: 0.5953 - val_loss: 1.1001 - val_accuracy: 0.6472\n",
      "Epoch 5/20\n",
      "26270/26270 [==============================] - 11s 403us/step - loss: 0.9881 - accuracy: 0.6978 - val_loss: 0.8241 - val_accuracy: 0.7542\n",
      "Epoch 6/20\n",
      "26270/26270 [==============================] - 11s 402us/step - loss: 0.8455 - accuracy: 0.7402 - val_loss: 0.7181 - val_accuracy: 0.7850\n",
      "Epoch 7/20\n",
      "26270/26270 [==============================] - 11s 409us/step - loss: 0.6265 - accuracy: 0.8114 - val_loss: 0.5727 - val_accuracy: 0.8215\n",
      "Epoch 8/20\n",
      "26270/26270 [==============================] - 11s 420us/step - loss: 0.5260 - accuracy: 0.8411 - val_loss: 0.4642 - val_accuracy: 0.8573\n",
      "Epoch 9/20\n",
      "26270/26270 [==============================] - 11s 406us/step - loss: 0.4427 - accuracy: 0.8681 - val_loss: 0.4523 - val_accuracy: 0.8627\n",
      "Epoch 10/20\n",
      "26270/26270 [==============================] - 11s 407us/step - loss: 0.5146 - accuracy: 0.8428 - val_loss: 0.5689 - val_accuracy: 0.8273\n",
      "Epoch 11/20\n",
      "26270/26270 [==============================] - 11s 407us/step - loss: 0.5306 - accuracy: 0.8371 - val_loss: 0.5393 - val_accuracy: 0.8338\n",
      "Epoch 12/20\n",
      "26270/26270 [==============================] - 11s 407us/step - loss: 0.5537 - accuracy: 0.8301 - val_loss: 0.5069 - val_accuracy: 0.8392\n",
      "Epoch 13/20\n",
      "26270/26270 [==============================] - 11s 410us/step - loss: 0.5219 - accuracy: 0.8390 - val_loss: 0.5471 - val_accuracy: 0.8306\n",
      "Epoch 14/20\n",
      "26270/26270 [==============================] - 11s 408us/step - loss: 0.4955 - accuracy: 0.8467 - val_loss: 0.4852 - val_accuracy: 0.8508\n",
      "Epoch 15/20\n",
      "26270/26270 [==============================] - 11s 407us/step - loss: 0.4962 - accuracy: 0.8447 - val_loss: 0.3162 - val_accuracy: 0.9029\n",
      "Epoch 16/20\n",
      "26270/26270 [==============================] - 11s 411us/step - loss: 0.2732 - accuracy: 0.9163 - val_loss: 0.2558 - val_accuracy: 0.9233\n",
      "Epoch 17/20\n",
      "26270/26270 [==============================] - 11s 403us/step - loss: 0.2750 - accuracy: 0.9160 - val_loss: 0.3883 - val_accuracy: 0.8792\n",
      "Epoch 18/20\n",
      "26270/26270 [==============================] - 11s 404us/step - loss: 0.2647 - accuracy: 0.9200 - val_loss: 0.3266 - val_accuracy: 0.8955\n",
      "Epoch 19/20\n",
      "26270/26270 [==============================] - 11s 406us/step - loss: 0.2014 - accuracy: 0.9412 - val_loss: 0.1883 - val_accuracy: 0.9446\n",
      "Epoch 20/20\n",
      "26270/26270 [==============================] - 10s 400us/step - loss: 0.2150 - accuracy: 0.9367 - val_loss: 0.1826 - val_accuracy: 0.9457\n",
      "========================================================================================================\n",
      "momentum: 0.25\n",
      "Train on 26270 samples, validate on 12939 samples\n",
      "Epoch 1/20\n",
      "26270/26270 [==============================] - 11s 409us/step - loss: 3.1108 - accuracy: 0.1561 - val_loss: 2.3255 - val_accuracy: 0.3122\n",
      "Epoch 2/20\n",
      "26270/26270 [==============================] - 11s 404us/step - loss: 2.0569 - accuracy: 0.3499 - val_loss: 1.8940 - val_accuracy: 0.3903\n",
      "Epoch 3/20\n",
      "26270/26270 [==============================] - 11s 408us/step - loss: 1.8025 - accuracy: 0.4124 - val_loss: 1.7029 - val_accuracy: 0.4445\n",
      "Epoch 4/20\n",
      "26270/26270 [==============================] - 11s 416us/step - loss: 1.6561 - accuracy: 0.4576 - val_loss: 1.5775 - val_accuracy: 0.4833\n",
      "Epoch 5/20\n",
      "26270/26270 [==============================] - 11s 419us/step - loss: 1.5524 - accuracy: 0.4980 - val_loss: 1.4268 - val_accuracy: 0.5436\n",
      "Epoch 6/20\n",
      "26270/26270 [==============================] - 14s 524us/step - loss: 1.3427 - accuracy: 0.5707 - val_loss: 1.1978 - val_accuracy: 0.6299\n",
      "Epoch 7/20\n",
      "26270/26270 [==============================] - 11s 430us/step - loss: 1.1073 - accuracy: 0.6574 - val_loss: 1.2643 - val_accuracy: 0.6303\n",
      "Epoch 8/20\n",
      "26270/26270 [==============================] - 11s 418us/step - loss: 0.9187 - accuracy: 0.7211 - val_loss: 0.9149 - val_accuracy: 0.7095\n",
      "Epoch 9/20\n",
      "26270/26270 [==============================] - 11s 419us/step - loss: 0.7825 - accuracy: 0.7628 - val_loss: 0.8088 - val_accuracy: 0.7492\n",
      "Epoch 10/20\n",
      "26270/26270 [==============================] - 11s 419us/step - loss: 0.6687 - accuracy: 0.7990 - val_loss: 0.7332 - val_accuracy: 0.7685\n",
      "Epoch 11/20\n",
      "26270/26270 [==============================] - 11s 422us/step - loss: 0.6122 - accuracy: 0.8176 - val_loss: 0.5250 - val_accuracy: 0.8447\n",
      "Epoch 12/20\n",
      "26270/26270 [==============================] - 11s 422us/step - loss: 0.5686 - accuracy: 0.8305 - val_loss: 0.5150 - val_accuracy: 0.8347\n",
      "Epoch 13/20\n",
      "26270/26270 [==============================] - 11s 423us/step - loss: 0.4529 - accuracy: 0.8676 - val_loss: 0.4595 - val_accuracy: 0.8644\n",
      "Epoch 14/20\n",
      "26270/26270 [==============================] - 11s 419us/step - loss: 0.4819 - accuracy: 0.8554 - val_loss: 0.4702 - val_accuracy: 0.8607\n",
      "Epoch 15/20\n",
      "26270/26270 [==============================] - 11s 422us/step - loss: 0.4920 - accuracy: 0.8528 - val_loss: 0.5145 - val_accuracy: 0.8487\n",
      "Epoch 16/20\n",
      "26270/26270 [==============================] - 11s 420us/step - loss: 0.4158 - accuracy: 0.8751 - val_loss: 0.3942 - val_accuracy: 0.8799\n",
      "Epoch 17/20\n",
      "26270/26270 [==============================] - 11s 419us/step - loss: 0.3847 - accuracy: 0.8847 - val_loss: 0.3792 - val_accuracy: 0.8842\n",
      "Epoch 18/20\n",
      "26270/26270 [==============================] - 12s 443us/step - loss: 0.3449 - accuracy: 0.8962 - val_loss: 0.3183 - val_accuracy: 0.9043\n",
      "Epoch 19/20\n",
      "26270/26270 [==============================] - 13s 478us/step - loss: 0.3143 - accuracy: 0.9048 - val_loss: 0.2834 - val_accuracy: 0.9132\n",
      "Epoch 20/20\n",
      "26270/26270 [==============================] - 13s 491us/step - loss: 0.2978 - accuracy: 0.9108 - val_loss: 0.2865 - val_accuracy: 0.9133\n",
      "========================================================================================================\n",
      "momentum: 0.125\n",
      "Train on 26270 samples, validate on 12939 samples\n",
      "Epoch 1/20\n",
      "26270/26270 [==============================] - 11s 436us/step - loss: 3.3309 - accuracy: 0.1107 - val_loss: 2.7168 - val_accuracy: 0.2615\n",
      "Epoch 2/20\n",
      "26270/26270 [==============================] - 12s 438us/step - loss: 2.2118 - accuracy: 0.3290 - val_loss: 1.9424 - val_accuracy: 0.3688\n",
      "Epoch 3/20\n",
      "26270/26270 [==============================] - 11s 422us/step - loss: 1.8712 - accuracy: 0.3966 - val_loss: 1.8511 - val_accuracy: 0.4210\n",
      "Epoch 4/20\n",
      "26270/26270 [==============================] - 12s 443us/step - loss: 1.7172 - accuracy: 0.4484 - val_loss: 1.6344 - val_accuracy: 0.4728\n",
      "Epoch 5/20\n",
      "26270/26270 [==============================] - 12s 451us/step - loss: 1.5560 - accuracy: 0.5110 - val_loss: 1.4286 - val_accuracy: 0.5578\n",
      "Epoch 6/20\n",
      "26270/26270 [==============================] - 12s 442us/step - loss: 1.3479 - accuracy: 0.5764 - val_loss: 1.1919 - val_accuracy: 0.6201\n",
      "Epoch 7/20\n",
      "26270/26270 [==============================] - 11s 420us/step - loss: 1.1042 - accuracy: 0.6599 - val_loss: 0.9896 - val_accuracy: 0.7170\n",
      "Epoch 8/20\n",
      "26270/26270 [==============================] - 12s 448us/step - loss: 0.8891 - accuracy: 0.7332 - val_loss: 0.8828 - val_accuracy: 0.7338\n",
      "Epoch 9/20\n",
      "26270/26270 [==============================] - 11s 412us/step - loss: 0.6984 - accuracy: 0.7963 - val_loss: 0.6764 - val_accuracy: 0.8000\n",
      "Epoch 10/20\n",
      "26270/26270 [==============================] - 11s 411us/step - loss: 0.6117 - accuracy: 0.8209 - val_loss: 0.5935 - val_accuracy: 0.8246\n",
      "Epoch 11/20\n",
      "26270/26270 [==============================] - 11s 419us/step - loss: 0.5742 - accuracy: 0.8351 - val_loss: 0.9645 - val_accuracy: 0.7048\n",
      "Epoch 12/20\n",
      "26270/26270 [==============================] - 11s 418us/step - loss: 0.5881 - accuracy: 0.8259 - val_loss: 0.4876 - val_accuracy: 0.8516\n",
      "Epoch 13/20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26270/26270 [==============================] - 11s 412us/step - loss: 0.4624 - accuracy: 0.8662 - val_loss: 0.5089 - val_accuracy: 0.8539\n",
      "Epoch 14/20\n",
      "26270/26270 [==============================] - 11s 414us/step - loss: 0.4664 - accuracy: 0.8651 - val_loss: 0.4147 - val_accuracy: 0.8805\n",
      "Epoch 15/20\n",
      "26270/26270 [==============================] - 11s 412us/step - loss: 0.3635 - accuracy: 0.8968 - val_loss: 0.3397 - val_accuracy: 0.9042\n",
      "Epoch 16/20\n",
      "26270/26270 [==============================] - 11s 411us/step - loss: 0.3372 - accuracy: 0.9046 - val_loss: 0.3590 - val_accuracy: 0.8949\n",
      "Epoch 17/20\n",
      "26270/26270 [==============================] - 11s 408us/step - loss: 0.2848 - accuracy: 0.9214 - val_loss: 0.2603 - val_accuracy: 0.9277\n",
      "Epoch 18/20\n",
      "26270/26270 [==============================] - 11s 413us/step - loss: 0.2688 - accuracy: 0.9254 - val_loss: 0.2604 - val_accuracy: 0.9287\n",
      "Epoch 19/20\n",
      "26270/26270 [==============================] - 11s 415us/step - loss: 0.2776 - accuracy: 0.9232 - val_loss: 0.4364 - val_accuracy: 0.8750\n",
      "Epoch 20/20\n",
      "26270/26270 [==============================] - 11s 421us/step - loss: 0.2477 - accuracy: 0.9306 - val_loss: 0.2296 - val_accuracy: 0.9362\n",
      "========================================================================================================\n",
      "momentum: 0.0625\n",
      "Train on 26270 samples, validate on 12939 samples\n",
      "Epoch 1/20\n",
      "26270/26270 [==============================] - 11s 420us/step - loss: 3.2931 - accuracy: 0.1228 - val_loss: 2.7893 - val_accuracy: 0.2639\n",
      "Epoch 2/20\n",
      "26270/26270 [==============================] - 11s 417us/step - loss: 2.3051 - accuracy: 0.3102 - val_loss: 2.0203 - val_accuracy: 0.3519\n",
      "Epoch 3/20\n",
      "26270/26270 [==============================] - 11s 418us/step - loss: 1.9448 - accuracy: 0.3750 - val_loss: 1.8662 - val_accuracy: 0.4032\n",
      "Epoch 4/20\n",
      "26270/26270 [==============================] - 11s 417us/step - loss: 1.8717 - accuracy: 0.4006 - val_loss: 2.1610 - val_accuracy: 0.3296\n",
      "Epoch 5/20\n",
      "26270/26270 [==============================] - 11s 437us/step - loss: 1.9170 - accuracy: 0.3950 - val_loss: 1.7780 - val_accuracy: 0.4216\n",
      "Epoch 6/20\n",
      "26270/26270 [==============================] - 12s 444us/step - loss: 1.7753 - accuracy: 0.4319 - val_loss: 1.7683 - val_accuracy: 0.4388\n",
      "Epoch 7/20\n",
      "26270/26270 [==============================] - 12s 450us/step - loss: 1.7000 - accuracy: 0.4527 - val_loss: 1.5991 - val_accuracy: 0.4785\n",
      "Epoch 8/20\n",
      "26270/26270 [==============================] - 12s 462us/step - loss: 1.5872 - accuracy: 0.4879 - val_loss: 1.5261 - val_accuracy: 0.5186\n",
      "Epoch 9/20\n",
      "26270/26270 [==============================] - 12s 465us/step - loss: 1.5079 - accuracy: 0.5211 - val_loss: 1.3463 - val_accuracy: 0.5678\n",
      "Epoch 10/20\n",
      "26270/26270 [==============================] - 11s 427us/step - loss: 1.4243 - accuracy: 0.5468 - val_loss: 1.3743 - val_accuracy: 0.5677\n",
      "Epoch 11/20\n",
      "26270/26270 [==============================] - 11s 434us/step - loss: 1.3981 - accuracy: 0.5548 - val_loss: 1.6236 - val_accuracy: 0.5027\n",
      "Epoch 12/20\n",
      "26270/26270 [==============================] - 11s 428us/step - loss: 1.2066 - accuracy: 0.6199 - val_loss: 1.3237 - val_accuracy: 0.5839\n",
      "Epoch 13/20\n",
      "26270/26270 [==============================] - 11s 413us/step - loss: 1.0687 - accuracy: 0.6650 - val_loss: 0.9750 - val_accuracy: 0.6896\n",
      "Epoch 14/20\n",
      "26270/26270 [==============================] - 11s 422us/step - loss: 0.9413 - accuracy: 0.7052 - val_loss: 0.9370 - val_accuracy: 0.7019\n",
      "Epoch 15/20\n",
      "26270/26270 [==============================] - 11s 413us/step - loss: 0.9215 - accuracy: 0.7164 - val_loss: 0.8350 - val_accuracy: 0.7436\n",
      "Epoch 16/20\n",
      "26270/26270 [==============================] - 11s 437us/step - loss: 0.7954 - accuracy: 0.7543 - val_loss: 0.9636 - val_accuracy: 0.6909\n",
      "Epoch 17/20\n",
      "26270/26270 [==============================] - 11s 432us/step - loss: 0.7465 - accuracy: 0.7691 - val_loss: 0.6481 - val_accuracy: 0.8039\n",
      "Epoch 18/20\n",
      "26270/26270 [==============================] - 11s 419us/step - loss: 0.6711 - accuracy: 0.7950 - val_loss: 0.6457 - val_accuracy: 0.7977\n",
      "Epoch 19/20\n",
      "26270/26270 [==============================] - 13s 496us/step - loss: 0.6973 - accuracy: 0.7911 - val_loss: 0.7226 - val_accuracy: 0.7675\n",
      "Epoch 20/20\n",
      "26270/26270 [==============================] - 11s 420us/step - loss: 0.6859 - accuracy: 0.7866 - val_loss: 0.6721 - val_accuracy: 0.7926\n",
      "========================================================================================================\n",
      "momentum: 0.03125\n",
      "Train on 26270 samples, validate on 12939 samples\n",
      "Epoch 1/20\n",
      "26270/26270 [==============================] - 11s 428us/step - loss: 3.0516 - accuracy: 0.1704 - val_loss: 2.2331 - val_accuracy: 0.3169\n",
      "Epoch 2/20\n",
      "26270/26270 [==============================] - 11s 427us/step - loss: 2.0234 - accuracy: 0.3595 - val_loss: 1.8524 - val_accuracy: 0.3897\n",
      "Epoch 3/20\n",
      "26270/26270 [==============================] - 11s 420us/step - loss: 1.7909 - accuracy: 0.4186 - val_loss: 1.7303 - val_accuracy: 0.4371\n",
      "Epoch 4/20\n",
      "26270/26270 [==============================] - 11s 423us/step - loss: 1.6831 - accuracy: 0.4558 - val_loss: 1.6722 - val_accuracy: 0.4678\n",
      "Epoch 5/20\n",
      "26270/26270 [==============================] - 11s 416us/step - loss: 1.5538 - accuracy: 0.5002 - val_loss: 1.5183 - val_accuracy: 0.5179\n",
      "Epoch 6/20\n",
      "26270/26270 [==============================] - 11s 424us/step - loss: 1.4138 - accuracy: 0.5495 - val_loss: 1.5096 - val_accuracy: 0.5125\n",
      "Epoch 7/20\n",
      "26270/26270 [==============================] - 11s 429us/step - loss: 1.2133 - accuracy: 0.6188 - val_loss: 1.1214 - val_accuracy: 0.6390\n",
      "Epoch 8/20\n",
      " 7936/26270 [========>.....................] - ETA: 6s - loss: 1.0794 - accuracy: 0.6628"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-12-7708ac0f9327>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[0mopt\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mkeras\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptimizers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mSGD\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlearning_rate\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0.125\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmomentum\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlearning_rates\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnesterov\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m     \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcompile\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'categorical_crossentropy'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptimizer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mopt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmetrics\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;34m'accuracy'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 6\u001b[1;33m     \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx_tr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_tr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mepochs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m20\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mx_tst\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_tst\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      7\u001b[0m     \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'========================================================================================================'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[0;32m   1237\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1238\u001b[0m                                         \u001b[0mvalidation_steps\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mvalidation_steps\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1239\u001b[1;33m                                         validation_freq=validation_freq)\n\u001b[0m\u001b[0;32m   1240\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1241\u001b[0m     def evaluate(self,\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\keras\\engine\\training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[1;34m(model, fit_function, fit_inputs, out_labels, batch_size, epochs, verbose, callbacks, val_function, val_inputs, shuffle, initial_epoch, steps_per_epoch, validation_steps, validation_freq)\u001b[0m\n\u001b[0;32m    194\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    195\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 196\u001b[1;33m                 \u001b[0mouts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfit_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    197\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    198\u001b[0m                 \u001b[1;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mo\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout_labels\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mouts\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\keras\\backend.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   3725\u001b[0m         \u001b[0mvalue\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmath_ops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcast\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtensor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3726\u001b[0m       \u001b[0mconverted_inputs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3727\u001b[1;33m     \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_graph_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mconverted_inputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3728\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3729\u001b[0m     \u001b[1;31m# EagerTensor.numpy() will often make a copy to ensure memory safety.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1549\u001b[0m       \u001b[0mTypeError\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mFor\u001b[0m \u001b[0minvalid\u001b[0m \u001b[0mpositional\u001b[0m\u001b[1;33m/\u001b[0m\u001b[0mkeyword\u001b[0m \u001b[0margument\u001b[0m \u001b[0mcombinations\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1550\u001b[0m     \"\"\"\n\u001b[1;32m-> 1551\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call_impl\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1552\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1553\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_call_impl\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, args, kwargs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1589\u001b[0m       raise TypeError(\"Keyword arguments {} unknown. Expected {}.\".format(\n\u001b[0;32m   1590\u001b[0m           list(kwargs.keys()), list(self._arg_keywords)))\n\u001b[1;32m-> 1591\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call_flat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcaptured_inputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1592\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1593\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_filtered_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[1;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1690\u001b[0m       \u001b[1;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1691\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[1;32m-> 1692\u001b[1;33m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[0;32m   1693\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[0;32m   1694\u001b[0m         \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36mcall\u001b[1;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[0;32m    543\u001b[0m               \u001b[0minputs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    544\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"executor_type\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mexecutor_type\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"config_proto\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 545\u001b[1;33m               ctx=ctx)\n\u001b[0m\u001b[0;32m    546\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    547\u001b[0m           outputs = execute.execute_with_cancellation(\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\eager\\execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[0;32m     59\u001b[0m     tensors = pywrap_tensorflow.TFE_Py_Execute(ctx._handle, device_name,\n\u001b[0;32m     60\u001b[0m                                                \u001b[0mop_name\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mattrs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 61\u001b[1;33m                                                num_outputs)\n\u001b[0m\u001b[0;32m     62\u001b[0m   \u001b[1;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     63\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for m in momentums:\n",
    "    print('momentum:', learning_rates[i])\n",
    "    model = build_model(activation = 'sigmoid', batch_normalization = False)  # try 'relu' too\n",
    "    opt = tf.keras.optimizers.SGD(learning_rate = 0.125, momentum = m, nesterov=False)\n",
    "    model.compile(loss = 'categorical_crossentropy', optimizer = opt, metrics = ['accuracy'])\n",
    "    model.fit(x_tr, y_tr, epochs = 20, validation_data= (x_tst, y_tst))\n",
    "    print('========================================================================================================')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "colab_type": "text",
    "id": "sRf_XhQeldqz",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 31367 samples, validate on 7842 samples\n",
      "Epoch 1/20\n",
      "31367/31367 [==============================] - 19s 612us/step - loss: 2.0890 - accuracy: 0.4423 - val_loss: 1.3371 - val_accuracy: 0.5959\n",
      "Epoch 2/20\n",
      "31367/31367 [==============================] - 19s 612us/step - loss: 1.0283 - accuracy: 0.7164 - val_loss: 0.7710 - val_accuracy: 0.7828\n",
      "Epoch 3/20\n",
      "31367/31367 [==============================] - 19s 612us/step - loss: 0.7648 - accuracy: 0.7876 - val_loss: 0.7003 - val_accuracy: 0.8021\n",
      "Epoch 4/20\n",
      "31367/31367 [==============================] - 19s 620us/step - loss: 0.6282 - accuracy: 0.8189 - val_loss: 1.5658 - val_accuracy: 0.5598\n",
      "Epoch 5/20\n",
      "31367/31367 [==============================] - 19s 620us/step - loss: 0.5488 - accuracy: 0.8405 - val_loss: 0.4858 - val_accuracy: 0.8616\n",
      "Epoch 6/20\n",
      "31367/31367 [==============================] - 19s 619us/step - loss: 0.4619 - accuracy: 0.8675 - val_loss: 0.6529 - val_accuracy: 0.7989\n",
      "Epoch 7/20\n",
      "31367/31367 [==============================] - 19s 618us/step - loss: 0.4474 - accuracy: 0.8730 - val_loss: 0.5735 - val_accuracy: 0.8322\n",
      "Epoch 8/20\n",
      "31367/31367 [==============================] - 19s 621us/step - loss: 0.4490 - accuracy: 0.8705 - val_loss: 0.3566 - val_accuracy: 0.8942\n",
      "Epoch 9/20\n",
      "31367/31367 [==============================] - 20s 634us/step - loss: 0.3887 - accuracy: 0.8883 - val_loss: 0.4773 - val_accuracy: 0.8574\n",
      "Epoch 10/20\n",
      "31367/31367 [==============================] - 20s 625us/step - loss: 0.3467 - accuracy: 0.9004 - val_loss: 0.4073 - val_accuracy: 0.8796\n",
      "Epoch 11/20\n",
      "31367/31367 [==============================] - 20s 632us/step - loss: 0.3254 - accuracy: 0.9051 - val_loss: 0.5338 - val_accuracy: 0.8298\n",
      "Epoch 12/20\n",
      "31367/31367 [==============================] - 20s 623us/step - loss: 0.2928 - accuracy: 0.9157 - val_loss: 0.4834 - val_accuracy: 0.8542\n",
      "Epoch 13/20\n",
      "31367/31367 [==============================] - 20s 634us/step - loss: 0.3126 - accuracy: 0.9070 - val_loss: 0.2967 - val_accuracy: 0.9138\n",
      "Epoch 14/20\n",
      "31367/31367 [==============================] - 20s 624us/step - loss: 0.2775 - accuracy: 0.9187 - val_loss: 0.3260 - val_accuracy: 0.8995\n",
      "Epoch 15/20\n",
      "31367/31367 [==============================] - 20s 624us/step - loss: 0.3820 - accuracy: 0.8890 - val_loss: 0.3239 - val_accuracy: 0.8995\n",
      "Epoch 16/20\n",
      "31367/31367 [==============================] - 20s 626us/step - loss: 0.2521 - accuracy: 0.9232 - val_loss: 0.2723 - val_accuracy: 0.9183\n",
      "Epoch 17/20\n",
      "31367/31367 [==============================] - 20s 644us/step - loss: 0.2531 - accuracy: 0.9256 - val_loss: 0.3080 - val_accuracy: 0.9084\n",
      "Epoch 18/20\n",
      "31367/31367 [==============================] - 20s 636us/step - loss: 0.2381 - accuracy: 0.9299 - val_loss: 0.3275 - val_accuracy: 0.8981\n",
      "Epoch 19/20\n",
      "31367/31367 [==============================] - 20s 639us/step - loss: 0.3031 - accuracy: 0.9096 - val_loss: 0.7796 - val_accuracy: 0.7666\n",
      "Epoch 20/20\n",
      "31367/31367 [==============================] - 20s 630us/step - loss: 0.2044 - accuracy: 0.9394 - val_loss: 0.2661 - val_accuracy: 0.9160\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x1be0a5e3e48>"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = build_model(activation = 'sigmoid', batch_normalization = True)  # try 'relu' too\n",
    "opt = tf.keras.optimizers.SGD(learning_rate = 0.1, momentum = 0, nesterov=True)\n",
    "model.compile(loss = 'categorical_crossentropy', optimizer = opt, metrics = ['accuracy'])\n",
    "model.fit(x_tr, y_tr, epochs = 20, validation_data= (x_tst, y_tst))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[7.3597480e-06 1.1880704e-06 4.0130180e-07 ... 9.2217283e-07\n",
      "  4.8400634e-06 8.4510185e-07]\n",
      " [1.4789744e-04 9.7446352e-01 1.5995530e-03 ... 7.1707946e-06\n",
      "  3.6577674e-08 4.9049589e-07]\n",
      " [1.3829646e-06 1.9243575e-07 3.6817747e-07 ... 7.9551297e-05\n",
      "  1.6468884e-05 1.5074387e-06]\n",
      " ...\n",
      " [3.5664780e-04 1.0475375e-03 1.5668098e-03 ... 8.8161137e-04\n",
      "  2.4159157e-04 4.5088156e-05]\n",
      " [1.3376473e-03 3.8526282e-03 5.0244057e-03 ... 5.3995554e-03\n",
      "  1.0639826e-05 3.9452375e-04]\n",
      " [5.7629490e-04 9.8339608e-03 1.2270998e-03 ... 2.1267186e-03\n",
      "  3.5112803e-03 2.1784692e-03]]\n",
      "12630\n",
      "38\n"
     ]
    }
   ],
   "source": [
    "yhat = model.predict(xtest0, verbose = False)\n",
    "\n",
    "print(yhat)\n",
    "print(len(yhat))\n",
    "\n",
    "\n",
    "y_TST = [np.argmax(yhat[i]) for i in range(len(yhat))]\n",
    "\n",
    "sub = [[0 for j in range(44)] for i in range(len(y_TST))]\n",
    "\n",
    "for i in range(len(y_TST)):\n",
    "    j = y_TST[i]+1\n",
    "    sub[i][0] = i\n",
    "    sub[i][j] = 1\n",
    "# print(min(y_TST))\n",
    "# print(max(y_TST))\n",
    "\n",
    "print(y_TST[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 31367 samples, validate on 7842 samples\n",
      "Epoch 1/100\n",
      "31367/31367 [==============================] - 20s 645us/step - loss: 3.6727 - accuracy: 0.0841 - val_loss: 3.4529 - val_accuracy: 0.1729\n",
      "Epoch 2/100\n",
      "31367/31367 [==============================] - 20s 638us/step - loss: 3.3240 - accuracy: 0.2169 - val_loss: 3.2240 - val_accuracy: 0.2437\n",
      "Epoch 3/100\n",
      "31367/31367 [==============================] - 20s 642us/step - loss: 3.1247 - accuracy: 0.2594 - val_loss: 3.0517 - val_accuracy: 0.2686\n",
      "Epoch 4/100\n",
      "31367/31367 [==============================] - 20s 643us/step - loss: 2.9623 - accuracy: 0.2944 - val_loss: 2.8987 - val_accuracy: 0.3044\n",
      "Epoch 5/100\n",
      "31367/31367 [==============================] - 20s 643us/step - loss: 2.8187 - accuracy: 0.3276 - val_loss: 2.7624 - val_accuracy: 0.3374\n",
      "Epoch 6/100\n",
      "31367/31367 [==============================] - 20s 645us/step - loss: 2.6932 - accuracy: 0.3510 - val_loss: 2.6442 - val_accuracy: 0.3590\n",
      "Epoch 7/100\n",
      "31367/31367 [==============================] - 20s 648us/step - loss: 2.5819 - accuracy: 0.3714 - val_loss: 2.5407 - val_accuracy: 0.3815\n",
      "Epoch 8/100\n",
      "31367/31367 [==============================] - 20s 648us/step - loss: 2.4821 - accuracy: 0.3868 - val_loss: 2.4524 - val_accuracy: 0.3915\n",
      "Epoch 9/100\n",
      "31367/31367 [==============================] - 20s 651us/step - loss: 2.3939 - accuracy: 0.4029 - val_loss: 2.3613 - val_accuracy: 0.4105\n",
      "Epoch 10/100\n",
      "31367/31367 [==============================] - 21s 659us/step - loss: 2.3147 - accuracy: 0.4213 - val_loss: 2.2888 - val_accuracy: 0.4264\n",
      "Epoch 11/100\n",
      "31367/31367 [==============================] - 20s 649us/step - loss: 2.2439 - accuracy: 0.4357 - val_loss: 2.2224 - val_accuracy: 0.4452\n",
      "Epoch 12/100\n",
      "31367/31367 [==============================] - 21s 657us/step - loss: 2.1795 - accuracy: 0.4484 - val_loss: 2.1587 - val_accuracy: 0.4573\n",
      "Epoch 13/100\n",
      "31367/31367 [==============================] - 20s 652us/step - loss: 2.1213 - accuracy: 0.4635 - val_loss: 2.0996 - val_accuracy: 0.4707\n",
      "Epoch 14/100\n",
      "31367/31367 [==============================] - 20s 653us/step - loss: 2.0682 - accuracy: 0.4740 - val_loss: 2.0549 - val_accuracy: 0.4756\n",
      "Epoch 15/100\n",
      "31367/31367 [==============================] - 21s 681us/step - loss: 2.0187 - accuracy: 0.4847 - val_loss: 2.0048 - val_accuracy: 0.4890\n",
      "Epoch 16/100\n",
      "31367/31367 [==============================] - 20s 646us/step - loss: 1.9717 - accuracy: 0.4957 - val_loss: 1.9599 - val_accuracy: 0.5009\n",
      "Epoch 17/100\n",
      "31367/31367 [==============================] - 20s 650us/step - loss: 1.9273 - accuracy: 0.5059 - val_loss: 1.9174 - val_accuracy: 0.5148\n",
      "Epoch 18/100\n",
      "31367/31367 [==============================] - 20s 650us/step - loss: 1.8861 - accuracy: 0.5153 - val_loss: 1.8760 - val_accuracy: 0.5268\n",
      "Epoch 19/100\n",
      "31367/31367 [==============================] - 20s 650us/step - loss: 1.8473 - accuracy: 0.5263 - val_loss: 1.8346 - val_accuracy: 0.5335\n",
      "Epoch 20/100\n",
      "31367/31367 [==============================] - 21s 656us/step - loss: 1.8085 - accuracy: 0.5370 - val_loss: 1.7991 - val_accuracy: 0.5449\n",
      "Epoch 21/100\n",
      "31367/31367 [==============================] - 21s 658us/step - loss: 1.7728 - accuracy: 0.5471 - val_loss: 1.7700 - val_accuracy: 0.5571\n",
      "Epoch 22/100\n",
      "31367/31367 [==============================] - 21s 667us/step - loss: 1.7379 - accuracy: 0.5590 - val_loss: 1.7309 - val_accuracy: 0.5615\n",
      "Epoch 23/100\n",
      "31367/31367 [==============================] - 21s 657us/step - loss: 1.7045 - accuracy: 0.5695 - val_loss: 1.7025 - val_accuracy: 0.5672\n",
      "Epoch 24/100\n",
      "31367/31367 [==============================] - 20s 653us/step - loss: 1.6721 - accuracy: 0.5805 - val_loss: 1.6662 - val_accuracy: 0.5858\n",
      "Epoch 25/100\n",
      "31367/31367 [==============================] - 21s 660us/step - loss: 1.6405 - accuracy: 0.5881 - val_loss: 1.6490 - val_accuracy: 0.5756\n",
      "Epoch 26/100\n",
      "31367/31367 [==============================] - 21s 655us/step - loss: 1.6105 - accuracy: 0.5976 - val_loss: 1.6180 - val_accuracy: 0.5877\n",
      "Epoch 27/100\n",
      "31367/31367 [==============================] - 21s 669us/step - loss: 1.5797 - accuracy: 0.6080 - val_loss: 1.5800 - val_accuracy: 0.6044\n",
      "Epoch 28/100\n",
      "31367/31367 [==============================] - 23s 725us/step - loss: 1.5501 - accuracy: 0.6168 - val_loss: 1.5499 - val_accuracy: 0.6125\n",
      "Epoch 29/100\n",
      "31367/31367 [==============================] - 21s 657us/step - loss: 1.5213 - accuracy: 0.6255 - val_loss: 1.5245 - val_accuracy: 0.6214\n",
      "Epoch 30/100\n",
      "31367/31367 [==============================] - 21s 668us/step - loss: 1.4929 - accuracy: 0.6328 - val_loss: 1.4933 - val_accuracy: 0.6269\n",
      "Epoch 31/100\n",
      "31367/31367 [==============================] - 22s 695us/step - loss: 1.4650 - accuracy: 0.6422 - val_loss: 1.4637 - val_accuracy: 0.6404\n",
      "Epoch 32/100\n",
      "31367/31367 [==============================] - 20s 653us/step - loss: 1.4389 - accuracy: 0.6500 - val_loss: 1.4405 - val_accuracy: 0.6524\n",
      "Epoch 33/100\n",
      "31367/31367 [==============================] - 21s 657us/step - loss: 1.4134 - accuracy: 0.6574 - val_loss: 1.4079 - val_accuracy: 0.6546\n",
      "Epoch 34/100\n",
      "31367/31367 [==============================] - 21s 660us/step - loss: 1.3844 - accuracy: 0.6647 - val_loss: 1.3849 - val_accuracy: 0.6648\n",
      "Epoch 35/100\n",
      "31367/31367 [==============================] - 21s 656us/step - loss: 1.3594 - accuracy: 0.6737 - val_loss: 1.3680 - val_accuracy: 0.6673\n",
      "Epoch 36/100\n",
      "31367/31367 [==============================] - 20s 636us/step - loss: 1.3316 - accuracy: 0.6817 - val_loss: 1.3370 - val_accuracy: 0.6779\n",
      "Epoch 37/100\n",
      "31367/31367 [==============================] - 20s 632us/step - loss: 1.3076 - accuracy: 0.6895 - val_loss: 1.3114 - val_accuracy: 0.6882\n",
      "Epoch 38/100\n",
      "31367/31367 [==============================] - 20s 638us/step - loss: 1.2821 - accuracy: 0.6982 - val_loss: 1.2925 - val_accuracy: 0.6936\n",
      "Epoch 39/100\n",
      "31367/31367 [==============================] - 20s 637us/step - loss: 1.2575 - accuracy: 0.7046 - val_loss: 1.2698 - val_accuracy: 0.6899\n",
      "Epoch 40/100\n",
      "31367/31367 [==============================] - 20s 633us/step - loss: 1.2353 - accuracy: 0.7107 - val_loss: 1.2335 - val_accuracy: 0.7080\n",
      "Epoch 41/100\n",
      "31367/31367 [==============================] - 20s 635us/step - loss: 1.2096 - accuracy: 0.7176 - val_loss: 1.2215 - val_accuracy: 0.7156\n",
      "Epoch 42/100\n",
      "31367/31367 [==============================] - 20s 640us/step - loss: 1.1869 - accuracy: 0.7248 - val_loss: 1.2031 - val_accuracy: 0.7233\n",
      "Epoch 43/100\n",
      "31367/31367 [==============================] - 20s 638us/step - loss: 1.1631 - accuracy: 0.7322 - val_loss: 1.1667 - val_accuracy: 0.7274\n",
      "Epoch 44/100\n",
      "31367/31367 [==============================] - 20s 642us/step - loss: 1.1390 - accuracy: 0.7392 - val_loss: 1.1581 - val_accuracy: 0.7298\n",
      "Epoch 45/100\n",
      "31367/31367 [==============================] - 20s 640us/step - loss: 1.1167 - accuracy: 0.7455 - val_loss: 1.1303 - val_accuracy: 0.7322\n",
      "Epoch 46/100\n",
      "31367/31367 [==============================] - 20s 639us/step - loss: 1.0943 - accuracy: 0.7511 - val_loss: 1.1060 - val_accuracy: 0.7428\n",
      "Epoch 47/100\n",
      "31367/31367 [==============================] - 20s 635us/step - loss: 1.0743 - accuracy: 0.7562 - val_loss: 1.0832 - val_accuracy: 0.7464\n",
      "Epoch 48/100\n",
      "31367/31367 [==============================] - 20s 637us/step - loss: 1.0545 - accuracy: 0.7615 - val_loss: 1.0630 - val_accuracy: 0.7572\n",
      "Epoch 49/100\n",
      "31367/31367 [==============================] - 20s 639us/step - loss: 1.0361 - accuracy: 0.7668 - val_loss: 1.0421 - val_accuracy: 0.7640\n",
      "Epoch 50/100\n",
      "31367/31367 [==============================] - 20s 639us/step - loss: 1.0129 - accuracy: 0.7744 - val_loss: 1.0301 - val_accuracy: 0.7599\n",
      "Epoch 51/100\n",
      "31367/31367 [==============================] - 20s 636us/step - loss: 0.9932 - accuracy: 0.7817 - val_loss: 1.0038 - val_accuracy: 0.7772\n",
      "Epoch 52/100\n",
      "31367/31367 [==============================] - 20s 638us/step - loss: 0.9755 - accuracy: 0.7858 - val_loss: 0.9909 - val_accuracy: 0.7728\n",
      "Epoch 53/100\n",
      "31367/31367 [==============================] - 20s 637us/step - loss: 0.9547 - accuracy: 0.7911 - val_loss: 0.9962 - val_accuracy: 0.7661\n",
      "Epoch 54/100\n",
      "31367/31367 [==============================] - 20s 635us/step - loss: 0.9368 - accuracy: 0.7968 - val_loss: 0.9614 - val_accuracy: 0.7863\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 55/100\n",
      "31367/31367 [==============================] - 20s 628us/step - loss: 0.9192 - accuracy: 0.8022 - val_loss: 0.9351 - val_accuracy: 0.7955\n",
      "Epoch 56/100\n",
      "31367/31367 [==============================] - 20s 632us/step - loss: 0.9034 - accuracy: 0.8073 - val_loss: 0.9420 - val_accuracy: 0.7812\n",
      "Epoch 57/100\n",
      "31367/31367 [==============================] - 20s 631us/step - loss: 0.8883 - accuracy: 0.8106 - val_loss: 0.9123 - val_accuracy: 0.8009\n",
      "Epoch 58/100\n",
      "31367/31367 [==============================] - 22s 687us/step - loss: 0.8701 - accuracy: 0.8157 - val_loss: 0.8870 - val_accuracy: 0.8064\n",
      "Epoch 59/100\n",
      "31367/31367 [==============================] - 20s 631us/step - loss: 0.8549 - accuracy: 0.8206 - val_loss: 0.8811 - val_accuracy: 0.8043\n",
      "Epoch 60/100\n",
      "31367/31367 [==============================] - 20s 631us/step - loss: 0.8404 - accuracy: 0.8250 - val_loss: 0.8548 - val_accuracy: 0.8203\n",
      "Epoch 61/100\n",
      "31367/31367 [==============================] - 20s 629us/step - loss: 0.8247 - accuracy: 0.8285 - val_loss: 0.8861 - val_accuracy: 0.7975\n",
      "Epoch 62/100\n",
      "31367/31367 [==============================] - 20s 633us/step - loss: 0.8092 - accuracy: 0.8331 - val_loss: 0.8277 - val_accuracy: 0.8249\n",
      "Epoch 63/100\n",
      "31367/31367 [==============================] - 20s 637us/step - loss: 0.7943 - accuracy: 0.8388 - val_loss: 0.8088 - val_accuracy: 0.8291\n",
      "Epoch 64/100\n",
      "31367/31367 [==============================] - 20s 627us/step - loss: 0.7834 - accuracy: 0.8414 - val_loss: 0.8009 - val_accuracy: 0.8312\n",
      "Epoch 65/100\n",
      "31367/31367 [==============================] - 20s 631us/step - loss: 0.7687 - accuracy: 0.8462 - val_loss: 0.7879 - val_accuracy: 0.8344\n",
      "Epoch 66/100\n",
      "31367/31367 [==============================] - 20s 646us/step - loss: 0.7546 - accuracy: 0.8488 - val_loss: 0.7868 - val_accuracy: 0.8313\n",
      "Epoch 67/100\n",
      "31367/31367 [==============================] - 21s 668us/step - loss: 0.7434 - accuracy: 0.8523 - val_loss: 0.7603 - val_accuracy: 0.8438\n",
      "Epoch 68/100\n",
      "31367/31367 [==============================] - 20s 634us/step - loss: 0.7296 - accuracy: 0.8559 - val_loss: 0.7490 - val_accuracy: 0.8502\n",
      "Epoch 69/100\n",
      "31367/31367 [==============================] - 20s 631us/step - loss: 0.7189 - accuracy: 0.8574 - val_loss: 0.7654 - val_accuracy: 0.8366\n",
      "Epoch 70/100\n",
      "31367/31367 [==============================] - 20s 631us/step - loss: 0.7075 - accuracy: 0.8615 - val_loss: 0.7304 - val_accuracy: 0.8525\n",
      "Epoch 71/100\n",
      "31367/31367 [==============================] - 20s 631us/step - loss: 0.6935 - accuracy: 0.8646 - val_loss: 0.7297 - val_accuracy: 0.8549\n",
      "Epoch 72/100\n",
      "31367/31367 [==============================] - 20s 631us/step - loss: 0.6845 - accuracy: 0.8684 - val_loss: 0.6986 - val_accuracy: 0.8599\n",
      "Epoch 73/100\n",
      "31367/31367 [==============================] - 21s 684us/step - loss: 0.6726 - accuracy: 0.8706 - val_loss: 0.6929 - val_accuracy: 0.8630\n",
      "Epoch 74/100\n",
      "31367/31367 [==============================] - 21s 660us/step - loss: 0.6614 - accuracy: 0.8742 - val_loss: 0.6848 - val_accuracy: 0.8609\n",
      "Epoch 75/100\n",
      "31360/31367 [============================>.] - ETA: 0s - loss: 0.6507 - accuracy: 0.8767"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-149-03813a3abf46>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[0mopt\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mkeras\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptimizers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mSGD\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlearning_rate\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m1e-5\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmomentum\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0.99\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnesterov\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcompile\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'categorical_crossentropy'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptimizer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mopt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmetrics\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;34m'accuracy'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx_tr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_tr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mepochs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m100\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mx_tst\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_tst\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[0;32m   1237\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1238\u001b[0m                                         \u001b[0mvalidation_steps\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mvalidation_steps\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1239\u001b[1;33m                                         validation_freq=validation_freq)\n\u001b[0m\u001b[0;32m   1240\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1241\u001b[0m     def evaluate(self,\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\keras\\engine\\training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[1;34m(model, fit_function, fit_inputs, out_labels, batch_size, epochs, verbose, callbacks, val_function, val_inputs, shuffle, initial_epoch, steps_per_epoch, validation_steps, validation_freq)\u001b[0m\n\u001b[0;32m    208\u001b[0m                                          \u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    209\u001b[0m                                          \u001b[0mcallbacks\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 210\u001b[1;33m                                          verbose=0)\n\u001b[0m\u001b[0;32m    211\u001b[0m                     \u001b[0mval_outs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mval_outs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    212\u001b[0m                     \u001b[1;31m# Same labels assumed.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\keras\\engine\\training_arrays.py\u001b[0m in \u001b[0;36mtest_loop\u001b[1;34m(model, f, ins, batch_size, verbose, steps, callbacks)\u001b[0m\n\u001b[0;32m    447\u001b[0m             \u001b[0mbatch_logs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;34m'batch'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mbatch_index\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'size'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch_ids\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    448\u001b[0m             \u001b[0mcallbacks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call_batch_hook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'test'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'begin'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_index\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 449\u001b[1;33m             \u001b[0mbatch_outs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    450\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    451\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mbatch_index\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\keras\\backend.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   3725\u001b[0m         \u001b[0mvalue\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmath_ops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcast\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtensor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3726\u001b[0m       \u001b[0mconverted_inputs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3727\u001b[1;33m     \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_graph_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mconverted_inputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3728\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3729\u001b[0m     \u001b[1;31m# EagerTensor.numpy() will often make a copy to ensure memory safety.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1549\u001b[0m       \u001b[0mTypeError\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mFor\u001b[0m \u001b[0minvalid\u001b[0m \u001b[0mpositional\u001b[0m\u001b[1;33m/\u001b[0m\u001b[0mkeyword\u001b[0m \u001b[0margument\u001b[0m \u001b[0mcombinations\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1550\u001b[0m     \"\"\"\n\u001b[1;32m-> 1551\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call_impl\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1552\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1553\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_call_impl\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, args, kwargs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1589\u001b[0m       raise TypeError(\"Keyword arguments {} unknown. Expected {}.\".format(\n\u001b[0;32m   1590\u001b[0m           list(kwargs.keys()), list(self._arg_keywords)))\n\u001b[1;32m-> 1591\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call_flat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcaptured_inputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1592\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1593\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_filtered_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[1;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1690\u001b[0m       \u001b[1;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1691\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[1;32m-> 1692\u001b[1;33m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[0;32m   1693\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[0;32m   1694\u001b[0m         \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36mcall\u001b[1;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[0;32m    543\u001b[0m               \u001b[0minputs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    544\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"executor_type\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mexecutor_type\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"config_proto\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 545\u001b[1;33m               ctx=ctx)\n\u001b[0m\u001b[0;32m    546\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    547\u001b[0m           outputs = execute.execute_with_cancellation(\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\eager\\execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[0;32m     59\u001b[0m     tensors = pywrap_tensorflow.TFE_Py_Execute(ctx._handle, device_name,\n\u001b[0;32m     60\u001b[0m                                                \u001b[0mop_name\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mattrs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 61\u001b[1;33m                                                num_outputs)\n\u001b[0m\u001b[0;32m     62\u001b[0m   \u001b[1;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     63\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model = build_model(activation = 'sigmoid', batch_normalization = True)  # try 'relu' too\n",
    "opt = tf.keras.optimizers.SGD(learning_rate = 1e-5, momentum = 0.99, nesterov=True)\n",
    "model.compile(loss = 'categorical_crossentropy', optimizer = opt, metrics = ['accuracy'])\n",
    "model.fit(x_tr, y_tr, epochs = 100, validation_data= (x_tst, y_tst))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>oh_0</th>\n",
       "      <th>oh_1</th>\n",
       "      <th>oh_2</th>\n",
       "      <th>oh_3</th>\n",
       "      <th>oh_4</th>\n",
       "      <th>oh_5</th>\n",
       "      <th>oh_6</th>\n",
       "      <th>oh_7</th>\n",
       "      <th>oh_8</th>\n",
       "      <th>oh_9</th>\n",
       "      <th>...</th>\n",
       "      <th>oh_33</th>\n",
       "      <th>oh_34</th>\n",
       "      <th>oh_35</th>\n",
       "      <th>oh_36</th>\n",
       "      <th>oh_37</th>\n",
       "      <th>oh_38</th>\n",
       "      <th>oh_39</th>\n",
       "      <th>oh_40</th>\n",
       "      <th>oh_41</th>\n",
       "      <th>oh_42</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12625</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12626</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12627</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12628</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12629</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12630 rows × 43 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       oh_0  oh_1  oh_2  oh_3  oh_4  oh_5  oh_6  oh_7  oh_8  oh_9  ...  oh_33  \\\n",
       "id                                                                 ...          \n",
       "0         0     0     0     0     0     0     0     0     0     0  ...      0   \n",
       "1         0     1     0     0     0     0     0     0     0     0  ...      0   \n",
       "2         0     0     0     0     0     0     0     0     0     0  ...      0   \n",
       "3         0     0     0     0     0     0     0     0     0     0  ...      1   \n",
       "4         0     0     0     0     0     0     0     0     0     0  ...      0   \n",
       "...     ...   ...   ...   ...   ...   ...   ...   ...   ...   ...  ...    ...   \n",
       "12625     0     0     0     0     0     0     0     0     0     0  ...      0   \n",
       "12626     0     0     0     0     0     0     0     0     0     0  ...      1   \n",
       "12627     0     0     0     0     0     1     0     0     0     0  ...      0   \n",
       "12628     0     0     0     0     0     0     0     1     0     0  ...      0   \n",
       "12629     0     0     0     0     0     0     0     0     0     0  ...      0   \n",
       "\n",
       "       oh_34  oh_35  oh_36  oh_37  oh_38  oh_39  oh_40  oh_41  oh_42  \n",
       "id                                                                    \n",
       "0          0      0      0      0      0      0      0      0      0  \n",
       "1          0      0      0      0      0      0      0      0      0  \n",
       "2          0      0      0      0      1      0      0      0      0  \n",
       "3          0      0      0      0      0      0      0      0      0  \n",
       "4          0      0      0      0      0      0      0      0      0  \n",
       "...      ...    ...    ...    ...    ...    ...    ...    ...    ...  \n",
       "12625      0      0      0      0      0      0      0      0      0  \n",
       "12626      0      0      0      0      0      0      0      0      0  \n",
       "12627      0      0      0      0      0      0      0      0      0  \n",
       "12628      0      0      0      0      0      0      0      0      0  \n",
       "12629      0      0      0      0      0      0      0      0      0  \n",
       "\n",
       "[12630 rows x 43 columns]"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "cols = []\n",
    "for i in range(43):\n",
    "    cols.append('oh_'+str(i))\n",
    "cols = ['id'] + cols\n",
    "\n",
    "df = pd.DataFrame(data=sub, columns=cols)\n",
    "df = df.set_index('id')\n",
    "\n",
    "df.to_csv('results.csv', sep=',', encoding='utf-8')\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results\n",
    "\n",
    "\n",
    "learning rate: 0.0001\n",
    "\n",
    "momentum: 0.99\n",
    "\n",
    "number epochs: 200\n",
    "\n",
    "Although the number of epochs is a bit high, having such a large epoch range allows our model to slowly increase accuracy as the epoch step increases. If we limit our number of epochs to 20, our model settles around 87% accuracy. If we increase epochs to 50, our model accuracy incrases to 90% or so. Increasing it further to 100 brings us close to 95% accuracy. Finally, 200 epochs brings us close to 97% accuracy.\n",
    "\n",
    "Overall, these results are similar to previous hyperparamter tuning we've performed in the past. By setting a low learning rate (with decent momentum to keep our model \"learning\" through each epoch step) and allowing our model to run through a high number of epochs, our model can steadily reduce its error until its performance plateaus."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "authorship_tag": "ABX9TyPmJA4hBfRBdrqRdmH05WdY",
   "mount_file_id": "19D6oAusolyAqbiz0L7_PIHRSMv0eAsTz",
   "name": "Copy of german_signs3.ipynb",
   "provenance": [
    {
     "file_id": "1Ys1JBCEmz69eskWG7x-PFnjZcyaSUlAx",
     "timestamp": 1586798975859
    },
    {
     "file_id": "1rgH0lPxAacqXN-WBthfm6MoRTyzuga_J",
     "timestamp": 1586746443733
    }
   ],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}