CNN-from-first-principle.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "%matplotlib inline\n",
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from functions import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Buiding components"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Linear\n",
    "Linear class is a fully-connected dense layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[-0.        , -0.        ],\n",
       "        [-0.        ,  0.02229694]]),\n",
       " (2, 2))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class Linear:\n",
    "    \"\"\"\n",
    "    Typical linear layer: y = Wx + b \n",
    "    \"\"\"\n",
    "    def __init__(self, in_feature, out_feature, bias=True):\n",
    "        \n",
    "        self.W = he_normal((in_feature, out_feature), fan_in=in_feature)\n",
    "        self.b = np.zeros((1, out_feature))\n",
    "        \n",
    "        self.bias = bias\n",
    "        self.db = np.zeros((1, out_feature))\n",
    "        \n",
    "    def forward(self, A_prev):\n",
    "        self.A_prev = A_prev\n",
    "        \n",
    "        Z = A_prev @ self.W  + self.b\n",
    "        \n",
    "        # relu\n",
    "        self.A = Z*(Z > 0)\n",
    "        \n",
    "        return self.A\n",
    "\n",
    "    def backward(self, dZ_prev):\n",
    "\n",
    "        A_prev = self.A_prev\n",
    "\n",
    "        m = len(self.A)\n",
    "\n",
    "\n",
    "        \n",
    "        # relu flow back from dZ: dZ*1 if Z>0, dZ*0 otherwise\n",
    "        self.dA = dZ_prev*(self.A > 0)\n",
    "\n",
    "        self.dW = A_prev.T @ self.dA\n",
    "        if self.bias: # update db only if True, else keep it zeros\n",
    "            self.db = self.dA.sum(axis=0, keepdims=True)\n",
    "        \n",
    "        self.dZ = self.dA @ self.W.T\n",
    "        \n",
    "        return self.dZ\n",
    "\n",
    "# Test\n",
    "X = np.random.randn(2,4)\n",
    "net = Linear(4,2)\n",
    "A = net.forward(X)\n",
    "A, A.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Linear Softmax\n",
    "Linear softmax is a fully connected layer follow by a softmax layer. This will be the last layer of the network that produce probabilites of class\n",
    "<img src=\"images/linear_sofmax.png\" width=\"400\"/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.9685024  0.0314976 ]\n",
      " [0.99629025 0.00370975]]\n"
     ]
    }
   ],
   "source": [
    "class Linear_SoftMax:\n",
    "    \"\"\"\n",
    "    This combines 2 layers: linear layer followed by a softmax layer.\n",
    "    Should not init with he_normal cause we dont use relu here, but probably won't hurt since it's the last layer.\n",
    "    \"\"\"\n",
    "    def __init__(self,in_feature,out_feature):\n",
    "        self.W = he_normal((in_feature, out_feature), fan_in=in_feature)\n",
    "        self.b = np.zeros((1, out_feature))\n",
    "        \n",
    "    def forward(self, A_prev):\n",
    "        self.A_prev = A_prev\n",
    "\n",
    "        Z = A_prev @ self.W + self.b\n",
    "        \n",
    "        # numerical stable softmax\n",
    "        expZ = np.exp(Z - np.max(Z, axis=-1, keepdims=True))\n",
    "        self.A = expZ/np.sum(expZ,axis=-1 , keepdims=True)\n",
    "        \n",
    "        \n",
    "        return self.A\n",
    "\n",
    "    def backward(self, dZ_prev):\n",
    "        # dZ_prev is flowed backward from loss function\n",
    "        m = len(self.A)\n",
    "        \n",
    "        A_prev = self.A_prev\n",
    "        \n",
    "        self.dA = dZ_prev\n",
    "        \n",
    "        self.dW = (A_prev.T @ self.dA)\n",
    "        self.db = np.sum(self.dA, axis=0, keepdims=True)\n",
    "        \n",
    "        self.dZ = self.dA @ self.W.T\n",
    "        \n",
    "        return self.dZ\n",
    "# Test\n",
    "X = np.random.randn(2,4)\n",
    "net = Linear_SoftMax(4,2)\n",
    "print(net.forward(X))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Flatten layer\n",
    "After several layers of convolution , we need to flatten result so we can feed into dense layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[ 0.81211276, -0.15301989,  1.99501214,  0.23879151],\n",
       "        [ 0.02822408,  1.20002538, -0.50066546,  0.57389875]]),\n",
       " (2, 4))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class Flatten:\n",
    "    \"\"\"\n",
    "    Flatten the input into 2D array (or 2D tensor if you insist)\n",
    "    \"\"\"\n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    def forward(self,Z):\n",
    "        self.A_prev_shape = Z.shape\n",
    "        \n",
    "        self.Z = Z.reshape(Z.shape[0],np.prod(Z.shape[1:]))\n",
    "        \n",
    "        return self.Z\n",
    "\n",
    "    def backward(self, dZ_prev):\n",
    "        \n",
    "        self.dZ = dZ_prev.reshape(self.A_prev_shape)\n",
    "\n",
    "        return self.dZ\n",
    "\n",
    "# Test\n",
    "X = np.random.randn(2,2,2,1)\n",
    "net = Flatten()\n",
    "A = net.forward(X)\n",
    "A, A.shape\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pooling layer\n",
    "We can use max-pooling or average pooling to reduce dimension H-W of output\n",
    "\n",
    "<img src='images/maxpool.png' width=400>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[[[1.23486392],\n",
       "          [1.32683471]],\n",
       " \n",
       "         [[1.736443  ],\n",
       "          [1.736443  ]]],\n",
       " \n",
       " \n",
       "        [[[1.12646682],\n",
       "          [1.56202237]],\n",
       " \n",
       "         [[1.68769072],\n",
       "          [2.26583694]]]]),\n",
       " (2, 2, 2, 1))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class Pooling:\n",
    "    \"\"\"\n",
    "    Implements the pooling layer with max or average pooling.\n",
    "    Max pooling put a filter over input in sliding window fasion, take the maximum cell in the filter\n",
    "     and put it into the output\n",
    "    This implementation mostly taken from coursera deeplearning.ai course\n",
    "    \n",
    "    Arguments:\n",
    "        A_prev -- Input data, numpy array of shape (m, n_H_prev, n_W_prev, n_C_prev)\n",
    "        hparameters -- python dictionary containing \"f\" and \"stride\"\n",
    "        mode -- the pooling mode you would like to use, defined as a string (\"max\" or \"average\")\n",
    "        \n",
    "        Returns:\n",
    "        A -- output of the pool layer, a numpy array of shape (m, n_H, n_W, n_C)\n",
    "    \"\"\"\n",
    "    def __init__(self, f, stride, mode='max'):\n",
    "        self.f = f\n",
    "        self.stride = stride\n",
    "        self.mode = mode\n",
    "    \n",
    "    def forward(self, A_prev):\n",
    "\n",
    "        # Retrieve hyperparameters from \"hparameters\"\n",
    "        f = self.f\n",
    "        stride = self.stride\n",
    "        mode = self.mode\n",
    "        \n",
    "        self.A_prev = A_prev\n",
    "\n",
    "        # Retrieve dimensions from the input shape\n",
    "        (m, n_H_prev, n_W_prev, n_C_prev) = A_prev.shape\n",
    "        \n",
    "\n",
    "        \n",
    "        # Define the dimensions of the output\n",
    "        n_H = int(1 + (n_H_prev - f) / stride)\n",
    "        n_W = int(1 + (n_W_prev - f) / stride)\n",
    "        n_C = n_C_prev\n",
    "        \n",
    "        # Initialize output matrix A\n",
    "        A = np.zeros((m, n_H, n_W, n_C))              \n",
    "        \n",
    "        for i in range(m):                         # loop over the training examples\n",
    "            for h in range(n_H):                     # loop on the vertical axis of the output volume\n",
    "    #             Find the vertical start and end of the current \"slice\" \n",
    "                vert_start = h*stride\n",
    "                vert_end = vert_start + f\n",
    "                \n",
    "                for w in range(n_W):                 # loop on the horizontal axis of the output volume\n",
    "    #                 Find the vertical start and end of the current \"slice\" \n",
    "                    horiz_start = w*stride\n",
    "                    horiz_end = horiz_start + f\n",
    "                    \n",
    "                    for c in range (n_C):            # loop over the channels of the output volume\n",
    "                        \n",
    "    #                     Use the corners to define the current slice on the ith training example of A_prev, channel c.\n",
    "                        a_prev_slice = A_prev[i]\n",
    "                        \n",
    "    #                     Compute the pooling operation on the slice. \n",
    "    #                     Use an if statement to differentiate the modes. \n",
    "    #                     Use np.max and np.mean.\n",
    "                        if mode == \"max\":\n",
    "                            A[i, h, w, c] = np.amax(a_prev_slice[vert_start:vert_end, horiz_start:horiz_end, c])\n",
    "                        elif mode == \"average\":\n",
    "                            A[i, h, w, c] = np.mean(a_prev_slice[vert_start:vert_end, horiz_start:horiz_end, c])\n",
    "        \n",
    "        # Making sure your output shape is correct\n",
    "        assert(A.shape == (m, n_H, n_W, n_C))\n",
    "\n",
    "        self.A = A\n",
    "\n",
    "        return self.A\n",
    "\n",
    "    def create_mask_from_window(self, x):\n",
    "        \"\"\"\n",
    "        Creates a mask from an input matrix x, to identify the max entry of x.\n",
    "        \n",
    "        Arguments:\n",
    "        x -- Array of shape (f, f)\n",
    "        \n",
    "        Returns:\n",
    "        mask -- Array of the same shape as window, contains a True at the position corresponding to the max entry of x.\n",
    "        \"\"\"    \n",
    "        mask = (x==np.max(x))\n",
    "\n",
    "        return mask\n",
    "\n",
    "    def distribute_value(self, dz, shape):\n",
    "        \"\"\"\n",
    "        Distributes the input value in the matrix of dimension shape\n",
    "        \n",
    "        Arguments:\n",
    "        dz -- input scalar\n",
    "        shape -- the shape (n_H, n_W) of the output matrix for which we want to distribute the value of dz\n",
    "        \n",
    "        Returns:\n",
    "        a -- Array of size (n_H, n_W) for which we distributed the value of dz\n",
    "        \"\"\"    \n",
    "        # Retrieve dimensions from shape\n",
    "        (n_H, n_W) = shape\n",
    "        \n",
    "        # Compute the value to distribute on the matrix \n",
    "        average = dz/(n_H*n_W)\n",
    "        \n",
    "        # Create a matrix where every entry is the \"average\" value \n",
    "        a = np.zeros((n_H, n_W))+average\n",
    "\n",
    "        return a\n",
    "\n",
    "    def backward(self, dA, mode = \"max\"):\n",
    "        \"\"\"\n",
    "        Implements the backward pass of the pooling layer\n",
    "\n",
    "        Arguments:\n",
    "        dA -- gradient of cost with respect to the output of the pooling layer, same shape as A\n",
    "        mode -- the pooling mode you would like to use, defined as a string (\"max\" or \"average\")\n",
    "\n",
    "        Returns:\n",
    "        dA_prev -- gradient of cost with respect to the input of the pooling layer, same shape as A_prev\n",
    "        \"\"\"\n",
    "        A_prev= self.A_prev\n",
    "        stride = self.stride\n",
    "        f = self.f\n",
    "\n",
    "        m, n_H_prev, n_W_prev, n_C_prev = A_prev.shape\n",
    "        m, n_H, n_W, n_C = dA.shape\n",
    "\n",
    "        dA_prev = np.zeros(A_prev.shape)\n",
    "\n",
    "        for i in range(m): # loop over the training examples\n",
    "\n",
    "            a_prev = A_prev[i]\n",
    "\n",
    "            for h in range(n_H):                   # loop on the vertical axis\n",
    "                for w in range(n_W):               # loop on the horizontal axis\n",
    "                    for c in range(n_C):           # loop over the channels (depth)\n",
    "\n",
    "                        vert_start = h*stride \n",
    "                        vert_end = vert_start + f\n",
    "                        horiz_start = w*stride\n",
    "                        horiz_end = horiz_start + f\n",
    "\n",
    "\n",
    "                        if mode == \"max\":\n",
    "\n",
    "                            a_prev_slice = a_prev[vert_start: vert_end, horiz_start:horiz_end, c]\n",
    "\n",
    "                            mask = self.create_mask_from_window(a_prev_slice)\n",
    "\n",
    "                            dA_prev[i, vert_start: vert_end, horiz_start: horiz_end, c] += mask*dA[i, h, w, c]\n",
    "\n",
    "                        elif mode == \"average\":\n",
    "\n",
    "                            da = dA[i, h, w, c]\n",
    "\n",
    "                            shape = (f,f)\n",
    "\n",
    "                            dA_prev[i, vert_start: vert_end, horiz_start: horiz_end, c] += self.distribute_value(da, shape)\n",
    "\n",
    "        assert(dA_prev.shape == A_prev.shape)\n",
    "        \n",
    "        self.dZ = dA_prev\n",
    "\n",
    "        return self.dZ\n",
    "    \n",
    "# Test\n",
    "X = np.random.randn(2,5,5,1)\n",
    "net = Pooling(f=3, stride=2, mode='max')\n",
    "A = net.forward(X)\n",
    "A, A.shape\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convolution layer\n",
    "The main component of CNN, it slides a filter over input dimension H and W, calculate the product filter x input then sum into the output.\n",
    "<img src='images/convolution-operation-14.png' width=500>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[[[ 0.94403741, -0.        ],\n",
       "          [-0.        ,  0.35572819],\n",
       "          [ 0.33408715,  0.3541151 ]],\n",
       " \n",
       "         [[-0.        ,  1.9870983 ],\n",
       "          [-0.        , -0.        ],\n",
       "          [ 1.65719765, -0.        ]],\n",
       " \n",
       "         [[-0.        , -0.        ],\n",
       "          [ 1.50316071, -0.        ],\n",
       "          [-0.        , -0.        ]]],\n",
       " \n",
       " \n",
       "        [[[ 0.31046488, -0.        ],\n",
       "          [ 0.06424836,  0.90388799],\n",
       "          [ 0.36075398,  0.09863601]],\n",
       " \n",
       "         [[-0.        ,  0.0473638 ],\n",
       "          [ 0.02029675,  0.15701879],\n",
       "          [ 0.89011194,  0.33130962]],\n",
       " \n",
       "         [[ 0.37800902,  0.24378408],\n",
       "          [-0.        , -0.        ],\n",
       "          [ 0.05497921,  0.33165276]]]]),\n",
       " (2, 3, 3, 2))"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class Conv2D:\n",
    "    \"\"\"\n",
    "    Convolution layer is similar to pooling layer, except taking cell with max value, we multiply filter \n",
    "    with input and sum the result. \n",
    "    \n",
    "    Apply one filter defined by parameters W on a single slice (a_slice_prev) of the output activation \n",
    "    of the previous layer.\n",
    "\n",
    "    Arguments:\n",
    "    a_slice_prev -- slice of input data of shape (f, f, n_C_prev)\n",
    "    W -- Weight parameters contained in a window - matrix of shape (f, f, n_C_prev)\n",
    "    b -- Bias parameters contained in a window - matrix of shape (1, 1, 1)\n",
    "\n",
    "    Returns:\n",
    "    Z -- a scalar value, the result of convolving the sliding window (W, b) on a slice x of the input data\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, in_chn, out_chn, kernel_size, in_shape, padding=0, stride=1, bias=True):\n",
    "        \n",
    "        fan_in = np.prod(in_shape)\n",
    "        \n",
    "        self.W = he_normal((kernel_size, kernel_size, in_chn, out_chn), fan_in=fan_in)\n",
    "        self.b = np.zeros((1,1,1,out_chn))\n",
    "        self.stride = stride\n",
    "        self.padding = padding\n",
    "        self.bias = bias\n",
    "\n",
    "\n",
    "    def zero_pad(self, X, pad):\n",
    "        \"\"\"\n",
    "        Pad with zeros all images of the dataset X. The padding is applied to the height and width of an image, \n",
    "        as illustrated in Figure 1.\n",
    "        \n",
    "        Argument:\n",
    "        X -- python numpy array of shape (m, n_H, n_W, n_C) representing a batch of m images\n",
    "        pad -- integer, amount of padding around each image on vertical and horizontal dimensions\n",
    "        \n",
    "        Returns:\n",
    "        X_pad -- padded image of shape (m, n_H + 2 * pad, n_W + 2 * pad, n_C)\n",
    "        \"\"\"\n",
    "        \n",
    "        X_pad = np.pad(X, ((0,0), (pad,pad), (pad,pad), (0,0)), mode='constant', constant_values = (0,0))\n",
    "\n",
    "        return X_pad\n",
    "\n",
    "\n",
    "    def forward(self, A_prev):\n",
    "        \"\"\"\n",
    "        Implements the forward propagation for a convolution function\n",
    "        \n",
    "        Arguments:\n",
    "        A_prev -- output activations of the previous layer, \n",
    "            numpy array of shape (m, n_H_prev, n_W_prev, n_C_prev)\n",
    "        W -- Weights, numpy array of shape (f, f, n_C_prev, n_C)\n",
    "        b -- Biases, numpy array of shape (1, 1, 1, n_C)\n",
    "        hparameters -- python dictionary containing \"stride\" and \"pad\"\n",
    "            \n",
    "        Returns:\n",
    "        Z -- conv output, numpy array of shape (m, n_H, n_W, n_C)\n",
    "        \"\"\"\n",
    "        self.A_prev = A_prev\n",
    "\n",
    "        W = self.W\n",
    "        b = self.b\n",
    "        stride = self.stride\n",
    "        pad = self.padding\n",
    "        biases = 0\n",
    "\n",
    "        # Retrieve dimensions from A_prev's shape (≈1 line)  \n",
    "        (m, n_H_prev, n_W_prev, n_C_prev) = A_prev.shape\n",
    "        \n",
    "\n",
    "        # Retrieve dimensions from W's shape (≈1 line)\n",
    "        (f, f, n_C_prev, n_C) = W.shape\n",
    "        \n",
    "\n",
    "        \n",
    "        # Compute the dimensions of the CONV output volume using the formula given above. \n",
    "        n_H = int((n_H_prev - f + 2*pad)/stride) + 1\n",
    "        n_W = int((n_W_prev - f + 2*pad)/stride) + 1\n",
    "        \n",
    "        # Initialize the output volume Z with zeros. \n",
    "        Z = np.zeros((m, n_H, n_W, n_C))\n",
    "        \n",
    "        \n",
    "        # Create A_prev_pad by padding A_prev\n",
    "        A_prev_pad = self.zero_pad(A_prev, pad)\n",
    "        \n",
    "        # create sliding window view into input\n",
    "        windowed_view = np.lib.stride_tricks.sliding_window_view(A_prev_pad, (f,f,n_C_prev), axis=(1,2,3))\n",
    "        windowed_view = windowed_view[:,::stride,::stride,...]\n",
    "        \n",
    "        # multiply with filter and sum into output\n",
    "        for c in range(n_C):               # loop over filters\n",
    "            out_mul = np.multiply(windowed_view, W[:,:,:,c])\n",
    "            out_sum = np.sum(out_mul,(-3,-2,-1)) # last 3 dimensions, quirk of stride_tricks\n",
    "            Z[:,:,:,c,None] = out_sum\n",
    "        # plus bias\n",
    "        Z = Z+b\n",
    "        \n",
    "        # relu forward\n",
    "        self.Z = Z*(Z>0)\n",
    "        \n",
    "        return self.Z\n",
    "\n",
    "    def backward(self, dZ):\n",
    "        \"\"\"\n",
    "        Implement the backward propagation for a convolution function\n",
    "        \n",
    "        Arguments:\n",
    "        dZ -- gradient of the cost with respect to the output of the conv layer (Z), numpy array of shape (m, n_H, n_W, n_C)\n",
    "        \n",
    "        Returns:\n",
    "        dA_prev -- gradient of the cost with respect to the input of the conv layer (A_prev),\n",
    "                   numpy array of shape (m, n_H_prev, n_W_prev, n_C_prev)\n",
    "        dW -- gradient of the cost with respect to the weights of the conv layer (W)\n",
    "              numpy array of shape (f, f, n_C_prev, n_C)\n",
    "        db -- gradient of the cost with respect to the biases of the conv layer (b)\n",
    "              numpy array of shape (1, 1, 1, n_C)\n",
    "        \"\"\"    \n",
    "        W = self.W\n",
    "        b = self.b\n",
    "        A_prev = self.A_prev\n",
    "\n",
    "        stride = self.stride\n",
    "        pad = self.padding\n",
    "            \n",
    "        # Retrieve dimensions from A_prev's shape\n",
    "        (m, n_H_prev, n_W_prev, n_C_prev) = A_prev.shape\n",
    "        # Retrieve dimensions from W's shape\n",
    "        (f, f, n_C_prev, n_C) = W.shape\n",
    "        \n",
    "\n",
    "        \n",
    "        # Retrieve dimensions from dZ's shape\n",
    "        (m, n_H, n_W, n_C) = dZ.shape\n",
    "        \n",
    "        # Initialize dA_prev, dW, db with the correct shapes\n",
    "        dA_prev = np.zeros(A_prev.shape)    \n",
    "        dW = np.zeros(W.shape)\n",
    "        db = np.zeros(b.shape)\n",
    "        \n",
    "        # Pad A_prev and dA_prev\n",
    "        A_prev_pad = self.zero_pad(A_prev, pad)\n",
    "        dA_prev_pad = self.zero_pad(dA_prev, pad)\n",
    "        \n",
    "        # relu flow backward\n",
    "        dZ = dZ*(self.Z > 0)\n",
    "        \n",
    "        for i in range(m):                       # loop over the training examples\n",
    "            \n",
    "            # select ith training example from A_prev_pad and dA_prev_pad\n",
    "            a_prev_pad = A_prev_pad[i]\n",
    "            da_prev_pad = dA_prev_pad[i]\n",
    "            \n",
    "            for h in range(n_H):                   # loop over vertical axis of the output volume\n",
    "                for w in range(n_W):               # loop over horizontal axis of the output volume\n",
    "                    for c in range(n_C):           # loop over the channels of the output volume\n",
    "                        \n",
    "        #                     Find the corners of the current \"slice\"\n",
    "                        vert_start = h*stride \n",
    "                        vert_end = vert_start + f\n",
    "                        horiz_start = w*stride\n",
    "                        horiz_end = horiz_start + f\n",
    "\n",
    "                        # Use the corners to define the slice from a_prev_pad\n",
    "                        a_slice = a_prev_pad[vert_start: vert_end, horiz_start:horiz_end, :]\n",
    "\n",
    "                        # gradient of sum (f*a_slide) = sum of gradients (f*dZ)\n",
    "                        da_prev_pad[vert_start:vert_end, horiz_start:horiz_end, :] += W[:,:,:,c] * dZ[i,h,w,c]\n",
    "                        dW[:,:,:,c] += a_slice * dZ[i,h,w,c]\n",
    "                        if self.bias: # update db only if True\n",
    "                            db[:,:,:,c] += dZ[i,h,w,c]\n",
    "                        \n",
    "            # Set the ith training example's dA_prev to the unpadded da_prev_pad (Hint: use X[pad:-pad, pad:-pad, :])\n",
    "            dA_prev[i, :, :, :] = da_prev_pad[pad:-pad,pad:-pad,:]\n",
    "        \n",
    "        # Making sure your output shape is correct\n",
    "        assert(dA_prev.shape == (m, n_H_prev, n_W_prev, n_C_prev))\n",
    "        \n",
    "        self.dZ = dA_prev\n",
    "        self.dW = dW\n",
    "        self.db = db\n",
    "        \n",
    "        return self.dZ\n",
    "\n",
    "# Test\n",
    "np.random.seed(12)\n",
    "X = np.random.randn(2,5,5,3)\n",
    "net = Conv2D(in_chn=3, out_chn=2, kernel_size=3, in_shape=X.shape[1:], padding=1, stride=2)\n",
    "A = net.forward(X)\n",
    "dZ = np.random.randn(*A.shape)\n",
    "net.backward(dZ)\n",
    "A, A.shape\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Batch normalization layer\n",
    "Normalize the batch using mean and standard deviation.There also scaling factor w and bias b that can be learned by network.\n",
    "\n",
    "<img src='standard_scaling.png' width=200>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[[[-1.58562844],\n",
       "          [ 2.02064677]],\n",
       " \n",
       "         [[ 0.76319475],\n",
       "          [ 0.97795103]]],\n",
       " \n",
       " \n",
       "        [[[-0.8522418 ],\n",
       "          [-0.54895796]],\n",
       " \n",
       "         [[-0.08314369],\n",
       "          [-0.59919913]]]]),\n",
       " (2, 2, 2, 1))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class BatchNorm:\n",
    "    \"\"\"\n",
    "    Standard normalization for batch. w*x_norm + b. The scaling w and shift b is learned by network.\n",
    "    \"\"\"\n",
    "    # normalize before activation\n",
    "    def __init__(self, input_shape):\n",
    "        \n",
    "        d = np.prod(input_shape)\n",
    "        \n",
    "        self.w = np.random.randn(1, d)\n",
    "        self.b = np.random.randn(1, d)\n",
    "    \n",
    "    def forward(self, x, eps=1e-7):\n",
    "        \n",
    "        shape = x.shape\n",
    "        x = x.reshape(x.shape[0], np.prod(x.shape[1:]))\n",
    "        \n",
    "        N, D = x.shape\n",
    "\n",
    "        #step1: calculate mean\n",
    "        mu = 1./N * np.sum(x, axis = 0)\n",
    "\n",
    "        #step2: subtract mean vector of every trainings example\n",
    "        xmu = x - mu\n",
    "\n",
    "        #step3: following the lower branch - calculation denominator\n",
    "        sq = xmu ** 2\n",
    "\n",
    "        #step4: calculate variance\n",
    "        var = 1./N * np.sum(sq, axis = 0)\n",
    "\n",
    "        #step5: add eps for numerical stability, then sqrt\n",
    "        sqrtvar = np.sqrt(var + eps)\n",
    "\n",
    "        #step6: invert sqrtwar\n",
    "        ivar = 1./sqrtvar\n",
    "\n",
    "        #step7: execute normalization\n",
    "        xhat = xmu * ivar\n",
    "\n",
    "        #step8: Nor the two transformation steps\n",
    "        Wx = self.w * xhat\n",
    "\n",
    "        #step9\n",
    "        out = Wx + self.b\n",
    "\n",
    "        #store intermediate\n",
    "        self.cache = (xhat,xmu,ivar,sqrtvar,var,eps)\n",
    "        \n",
    "        out = out.reshape(shape)\n",
    "\n",
    "        return out\n",
    "    \n",
    "    def backward(self, dout):\n",
    "        \n",
    "        shape = dout.shape\n",
    "        dout = dout.reshape(dout.shape[0], np.prod(dout.shape[1:]))\n",
    "        \n",
    "        #unfold the variables stored in cache\n",
    "        xhat,xmu,ivar,sqrtvar,var,eps = self.cache\n",
    "\n",
    "        #get the dimensions of the input/output\n",
    "        N,D = dout.shape\n",
    "\n",
    "        #step9\n",
    "        self.db = np.sum(dout, axis=0)\n",
    "        dWx = dout #not necessary, but more understandable\n",
    "\n",
    "        #step8\n",
    "        self.dW = np.sum(dWx*xhat, axis=0)\n",
    "        dxhat = dWx * self.w\n",
    "\n",
    "        #step7\n",
    "        divar = np.sum(dxhat*xmu, axis=0)\n",
    "        dxmu1 = dxhat * ivar\n",
    "\n",
    "        #step6\n",
    "        dsqrtvar = -1. /(sqrtvar**2) * divar\n",
    "\n",
    "        #step5\n",
    "        dvar = 0.5 * 1. /np.sqrt(var+eps) * dsqrtvar\n",
    "\n",
    "        #step4\n",
    "        dsq = 1. /N * np.ones((N,D)) * dvar\n",
    "\n",
    "        #step3\n",
    "        dxmu2 = 2 * xmu * dsq\n",
    "\n",
    "        #step2\n",
    "        dx1 = (dxmu1 + dxmu2)\n",
    "        dmu = -1 * np.sum(dxmu1+dxmu2, axis=0)\n",
    "\n",
    "        #step1\n",
    "        dx2 = 1. /N * np.ones((N,D)) * dmu\n",
    "\n",
    "        #step0\n",
    "        dZ = dx1 + dx2\n",
    "        \n",
    "        dZ = dZ.reshape(shape)\n",
    "        \n",
    "        return dZ\n",
    "\n",
    "# Test\n",
    "X = np.random.randn(2,2,2,1) + 1000\n",
    "bn = BatchNorm(X.shape[1:])\n",
    "A = bn.forward(X)\n",
    "A, A.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build the network\n",
    "We'll build CNN network from from the building blocks we've implemented\n",
    "### CNN network\n",
    "This network will have 2 layers of convolution with stride and batch normalization, then a flatten layer and dense output followed by softmax. I dont use pooling to save some computation time.\n",
    "<img src='images/cnn.png' width=500>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CNN:\n",
    "    \"\"\"\n",
    "    Typical pytorch style CNN network declaration.\n",
    "    \"\"\"\n",
    "    def __init__(self, in_shape, out_size):\n",
    "        \n",
    "        in_size = in_shape[0]\n",
    "        in_chn = in_shape[-1]\n",
    "        \n",
    "        conv1_channel = 12\n",
    "        self.convo1 = Conv2D(in_chn=in_chn, out_chn=conv1_channel, kernel_size=3, in_shape=in_shape, padding=1, stride=2, bias=False)\n",
    "#         self.pooling = Pooling(f=3, stride=2, mode='max')\n",
    "        \n",
    "        c1 = (in_size-3 + 2*1)//2 + 1\n",
    "        output_shape = (c1,c1,conv1_channel)\n",
    "        \n",
    "        self.batchnorm1 = BatchNorm(output_shape)\n",
    "        \n",
    "        conv2_channel = 2*conv1_channel\n",
    "        self.convo2 = Conv2D(in_chn=conv1_channel, out_chn=conv2_channel, kernel_size=3, in_shape=output_shape, padding=1, stride=2, bias=False)\n",
    "        \n",
    "        c2 = (c1-3 + 2*1)//2 + 1\n",
    "        output_shape = (c2,c2,conv2_channel)\n",
    "        \n",
    "        self.batchnorm2 = BatchNorm(output_shape)\n",
    "        \n",
    "        self.flatten = Flatten()\n",
    "        \n",
    "\n",
    "        linear_in = np.prod(output_shape)\n",
    "        \n",
    "        self.linear_softmax = Linear_SoftMax(linear_in, out_size)\n",
    "        \n",
    "        # only layers with trainable weights here, which are used in optimization/gradient update.\n",
    "        self.layers = {'convo1': self.convo1, 'batch_norm1': self.batchnorm1, 'convo2': self.convo2, \n",
    "                       \"batch_norm2\": self.batchnorm2, 'linear_softmax': self.linear_softmax}\n",
    "\n",
    "    def forward(self, X):\n",
    "        \n",
    "        X = self.convo1.forward(X)\n",
    "        X = self.batchnorm1.forward(X)\n",
    "        \n",
    "        X = self.convo2.forward(X)\n",
    "        X = self.batchnorm2.forward(X)\n",
    "\n",
    "        X = self.flatten.forward(X)\n",
    "        X = self.linear_softmax.forward(X)\n",
    "\n",
    "        return X\n",
    "    \n",
    "    def backward(self, dZ):\n",
    "        \n",
    "        dZ = self.linear_softmax.backward(dZ)\n",
    "        dZ = self.flatten.backward(dZ)\n",
    "        \n",
    "        dZ = self.batchnorm2.backward(dZ)\n",
    "        dZ = self.convo2.backward(dZ)\n",
    "\n",
    "        dZ = self.batchnorm1.backward(dZ)\n",
    "        dZ = self.convo1.backward(dZ)\n",
    "        \n",
    "        return dZ\n",
    "        \n",
    "    def set_weights(self, weight_list):\n",
    "        for k, (W,b) in weight_list.items():\n",
    "            self.layers[k].W = W\n",
    "            self.layers[k].b = b\n",
    "            \n",
    "    def get_weights(self):\n",
    "        return {k:(layer.W, layer.b) for k,layer in self.layers.items()}\n",
    "    \n",
    "    def get_dweights(self):\n",
    "        return {k:(layer.dW, layer.db) for k,layer in self.layers.items()}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.00616049 0.01636072 0.07160223 0.00899268 0.01024421 0.39133198\n",
      "  0.00306802 0.44568182 0.04259456 0.00396329]\n",
      " [0.13459887 0.01015649 0.09260266 0.05418247 0.05399731 0.44373704\n",
      "  0.00559607 0.07125395 0.11525929 0.01861585]]\n"
     ]
    }
   ],
   "source": [
    "# Test\n",
    "X = np.random.randn(2,8,8,1)\n",
    "y = np.random.randint(0,10,(2))\n",
    "Y = one_hot(y,10)\n",
    "net = CNN(in_shape=X.shape[1:], out_size=Y.shape[1])\n",
    "print(net.forward(X))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fully connected network\n",
    "For comparision sake, here is a declaration of simple fully-connected network\n",
    "\n",
    "<img src='images/cnn_fully_connected.png' width=400>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FC:\n",
    "    \"\"\"\n",
    "    Typical pytorch style implementation for fully-connected network.\n",
    "    Get and set weights is used for gradients checking and weights update\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, in_shape, out_size):\n",
    "        \n",
    "        in_size = np.prod(in_shape)\n",
    "        self.flatten = Flatten()\n",
    "        self.linear = Linear(in_size, in_size*2,bias=False)\n",
    "        self.bn = BatchNorm((in_size*2))\n",
    "        \n",
    "        self.linear_softmax = Linear_SoftMax(in_size*2, out_size)\n",
    "        \n",
    "        self.layers = {'linear': self.linear, 'batch_norm': self.bn,\n",
    "                       'linear_softmax': self.linear_softmax}\n",
    "\n",
    "    def forward(self, X):\n",
    "\n",
    "        X = self.flatten.forward(X)\n",
    "        X = self.linear.forward(X)\n",
    "        X = self.bn.forward(X)\n",
    "        X = self.linear_softmax.forward(X)\n",
    "\n",
    "        return X\n",
    "    \n",
    "    def backward(self, dZ):\n",
    "        \n",
    "        dZ = self.linear_softmax.backward(dZ)\n",
    "        dZ = self.bn.backward(dZ)\n",
    "        dZ = self.linear.backward(dZ)\n",
    "        dZ = self.flatten.backward(dZ)\n",
    "        \n",
    "        return dZ\n",
    "        \n",
    "    def set_weights(self, weight_list):\n",
    "        for k, (W,b) in weight_list.items():\n",
    "            self.layers[k].W = W\n",
    "            self.layers[k].b = b\n",
    "            \n",
    "    def get_weights(self):\n",
    "        return {k:(layer.W, layer.b) for k,layer in self.layers.items()}\n",
    "    \n",
    "    def get_dweights(self):\n",
    "        return {k:(layer.dW, layer.db) for k,layer in self.layers.items()}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.02035742 0.0498864  0.11718482 0.02379004 0.00078127 0.01678894\n",
      "  0.69979679 0.01433209 0.04968974 0.00739249]\n",
      " [0.07539637 0.02450412 0.19748246 0.00728455 0.03917388 0.38686909\n",
      "  0.19746237 0.0089252  0.0282873  0.03461464]]\n"
     ]
    }
   ],
   "source": [
    "# Test\n",
    "X = np.random.randn(2,8,8,1)\n",
    "y = np.random.randint(0,10,(2))\n",
    "Y = one_hot(y,10)\n",
    "\n",
    "net = FC(in_shape=X.shape[1:], out_size=Y.shape[1])\n",
    "print(net.forward(X))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loss functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Criteria:\n",
    "    \"\"\"\n",
    "    Some popular loss functions which are mean square error, binary cross entropy and categorical cross entropy.\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, mode):\n",
    "        self.mode = mode\n",
    "        \n",
    "    def forward(self, A, Y):\n",
    "        self.A_prev = A\n",
    "        self.Y = Y\n",
    "        \n",
    "        m = len(A)\n",
    "        \n",
    "        if self.mode == 'mse':\n",
    "            return 1/m * np.sum((Y - A)**2)\n",
    "        if self.mode == 'cross_entropy':\n",
    "            return 1/m * -np.sum(Y*np.log(A+1e-8))\n",
    "        if self.mode == 'binary_cross_entropy':\n",
    "            return 1/m * np.sum(-Y*np.log(A+1e-8) - (1-Y)*np.log(1-A+1e-8))\n",
    "        \n",
    "    def backward(self):\n",
    "        A = self.A_prev\n",
    "        Y = self.Y\n",
    "        m = len(A)\n",
    "        \n",
    "        if self.mode == 'mse':\n",
    "            self.dZ = 1/m * -2*(Y - A)\n",
    "        if self.mode == 'cross_entropy' or self.mode == 'binary_cross_entropy':\n",
    "            self.dZ = 1/m * (A - Y)\n",
    "            \n",
    "        return self.dZ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((2, 10), (2, 8, 8, 1))"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Test\n",
    "np.random.seed(12)\n",
    "n = 2\n",
    "X = np.random.randn(n,8,8,1)\n",
    "y = np.random.randint(0,10,(n,))\n",
    "Y = one_hot(y, 10)\n",
    "\n",
    "net = CNN(in_shape=X.shape[1:], out_size=Y.shape[1])\n",
    "criteria = Criteria(mode='cross_entropy')\n",
    "\n",
    "A = net.forward(X)\n",
    "loss = criteria.forward(A, Y)\n",
    "\n",
    "dZ = criteria.backward()\n",
    "dZ = net.backward(dZ)\n",
    "A.shape, dZ.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check the gradient\n",
    "The grad check implementation so that we know if our back prop implementations are corrects\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hepers that get the work done\n",
    "def vec_to_weights(net, weight_vec):\n",
    "    weight_list = {}\n",
    "    idx = 0\n",
    "\n",
    "    for k, (W,b) in net.get_weights().items():\n",
    "        weight_list[k]= (weight_vec[idx:idx+W.size].reshape(W.shape), \n",
    "                         weight_vec[idx+W.size:idx+W.size+b.size].reshape(b.shape))\n",
    "        idx += W.size + b.size\n",
    "        \n",
    "        \n",
    "    return weight_list\n",
    "\n",
    "def weights_to_vec(weight_list):\n",
    "    \n",
    "    return np.concatenate([np.concatenate([W.ravel(),b.ravel()]) for k, (W,b) in weight_list.items()]).reshape(-1,1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check if our helpers functions are correct\n",
    "weight_list1 = net.get_weights()\n",
    "weight_vec = weights_to_vec(weight_list1)\n",
    "weight_list2 = vec_to_weights(net, weight_vec)\n",
    "\n",
    "\n",
    "for k,W in weight_list2.items():\n",
    "    assert (weight_list1[k][0] == weight_list2[k][0]).all()\n",
    "    assert (weight_list1[k][1] == weight_list2[k][1]).all()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def grad_check(net, criteria, X, Y, epsilon1 = 1e-8):\n",
    "    \"\"\"\n",
    "    delta_W_approx = (J(W+epsilon) - J(W-epsilon))/(2*epsilon)\n",
    "    \"\"\"\n",
    "\n",
    "    weight_vec = weights_to_vec(net.get_weights())\n",
    "    W_modified = np.copy(weight_vec) # 1d vec\n",
    "    \n",
    "    dW_approx = np.zeros((weight_vec.size,1)) # 1d vec\n",
    "    \n",
    "    for idx in range(weight_vec.size): # in range vector size\n",
    "        temp = np.copy(W_modified[idx])\n",
    "\n",
    "        W_modified[idx] = temp + epsilon1\n",
    "        \n",
    "        net.set_weights(vec_to_weights(net, W_modified))\n",
    "        A = net.forward(X)\n",
    "        loss_plus = criteria.forward(A, Y)\n",
    "\n",
    "        W_modified[idx] = temp - epsilon1\n",
    "        \n",
    "        net.set_weights(vec_to_weights(net, W_modified))\n",
    "        A = net.forward(X)\n",
    "        loss_minus = criteria.forward(A, Y)\n",
    "\n",
    "\n",
    "        approx = (loss_plus - loss_minus)/(2*epsilon1)\n",
    "        dW_approx[idx] = approx # 1d vec\n",
    "    \n",
    "    dW_vec = weights_to_vec(net.get_dweights())\n",
    "    \n",
    "    diff = np.linalg.norm(dW_vec-dW_approx) / (np.linalg.norm(dW_vec)+np.linalg.norm(dW_approx))\n",
    "    print( f\"The difference is: {diff} (should be a small value)\")\n",
    "    \n",
    "    return dW_approx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(12)\n",
    "n = 2\n",
    "X = np.random.randn(n,8,8,1)\n",
    "y = np.random.randint(0,10,(n,))\n",
    "Y = one_hot(y, 10)\n",
    "\n",
    "criteria = Criteria(mode='cross_entropy')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Test gradient check**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = FC(in_shape=X.shape[1:], out_size=Y.shape[1])\n",
    "\n",
    "A = net.forward(X)\n",
    "loss = criteria.forward(A, Y)\n",
    "\n",
    "dZ = criteria.backward()\n",
    "dZ = net.backward(dZ)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The difference is: 3.4037558799347225e-06 (should be a small value)\n"
     ]
    }
   ],
   "source": [
    "dW_approx = grad_check(net, criteria, X, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = CNN(in_shape=X.shape[1:], out_size=Y.shape[1])\n",
    "\n",
    "A = net.forward(X)\n",
    "loss = criteria.forward(A, Y)\n",
    "\n",
    "dZ = criteria.backward()\n",
    "dZ = net.backward(dZ)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The difference is: 1.345707433834566e-05 (should be a small value)\n"
     ]
    }
   ],
   "source": [
    "dW_approx = grad_check(net, criteria, X, Y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Update gradidents\n",
    "The favourite Adam that combines momentum and rmsprop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Adam:\n",
    "    \"\"\"\n",
    "    Our favourite Adam optimization that combines momentum and rmsprop\n",
    "    \"\"\"\n",
    "    def __init__(self, net, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8):\n",
    "        # params\n",
    "        params = net.get_weights()\n",
    "        self.learning_rate = learning_rate\n",
    "\n",
    "        self.beta1=beta1\n",
    "        self.beta2=beta2\n",
    "        self.epsilon=epsilon\n",
    "        \n",
    "        # init\n",
    "        self.VW = {}\n",
    "        self.Vb = {}\n",
    "        self.SW = {}\n",
    "        self.Sb = {}\n",
    "        for k in params.keys():\n",
    "            self.VW[k] = 0\n",
    "            self.Vb[k] = 0\n",
    "            self.SW[k] = 0\n",
    "            self.Sb[k] = 0\n",
    "        \n",
    "    def step(self, net):\n",
    "        \n",
    "        params = net.get_weights()\n",
    "        dparams = net.get_dweights()\n",
    "        \n",
    "        beta1 = self.beta1\n",
    "        beta2 = self.beta2\n",
    "        \n",
    "        for k,(dW,db) in dparams.items():\n",
    "            W,b = params[k]\n",
    "            # momentum\n",
    "            self.VW[k] = beta1*self.VW[k] + (1.-beta1)*dW #dW\n",
    "            self.Vb[k] = beta1*self.Vb[k] + (1.-beta1)*db #db\n",
    "            # rmsprop\n",
    "            self.SW[k] = beta2*self.SW[k] + (1.-beta2)*(dW**2) #dW**2\n",
    "            self.Sb[k] = beta2*self.Sb[k] + (1.-beta2)*(db**2) #db**2\n",
    "            \n",
    "            W -= self.learning_rate * self.VW[k]/(np.sqrt(self.SW[k]) + self.epsilon) # W\n",
    "            b -= self.learning_rate * self.Vb[k]/(np.sqrt(self.Sb[k]) + self.epsilon) # b\n",
    "#             params[k] = (W,b)\n",
    "        \n",
    "        # no need set back weights to net  cause params is passed by reference\n",
    "#         net.set_weights(params)\n",
    "        \n",
    "    def set_lr(self,lr):\n",
    "        \n",
    "        self.learning_rate = lr\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.857179093899613\n",
      "4.393269147227144\n",
      "3.2005708092189833\n",
      "2.0294196374353843\n",
      "1.167980316767712\n",
      "0.7308159162056422\n",
      "0.4055687064587187\n",
      "0.1871264889908893\n",
      "0.08441719264125348\n",
      "0.03722542925081371\n"
     ]
    }
   ],
   "source": [
    "# check if loss decrease every step\n",
    "np.random.seed(12)\n",
    "n = 2\n",
    "X = np.random.randn(n,8,8,1)\n",
    "y = np.random.randint(0,10,(n,))\n",
    "Y = one_hot(y, 10)\n",
    "\n",
    "net = CNN(in_shape=X.shape[1:], out_size=Y.shape[1])\n",
    "optimizer = Adam(net,learning_rate=0.001)\n",
    "criteria = Criteria(mode='cross_entropy')\n",
    "\n",
    "for i in range(10):\n",
    "    A = net.forward(X)\n",
    "    \n",
    "    loss = criteria.forward(A, Y)\n",
    "    print(loss)\n",
    "\n",
    "    dZ = criteria.backward()\n",
    "    dZ = net.backward(dZ)\n",
    "\n",
    "    optimizer.step(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare data\n",
    "We use MNIST hand draw digit, and CIFAR10 10 object classes to test our network implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 400,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input size:(1438, 8, 8, 1) Number of images:1438, Classes:[0 1 2 3 4 5 6 7 8 9]\n",
      "number 8\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPUAAAD4CAYAAAA0L6C7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAALI0lEQVR4nO3df6jddR3H8dfL6+bc3BJNbeyOpuCWGuTkNrGV5JYxf6AFERspJMUEURwVov0TQvRPJAqluKYmuJSaDkWmJvkroZZ3PyrnnbKWtuvU6zBxau26+e6PewZTr93vOef749x3zwcM773ncD/vw3zue+73nvP9OCIEII/Dmh4AQLmIGkiGqIFkiBpIhqiBZA6v4ptO9RExTTOq+NaNGp1d72Oac+wbta01MjqztrX8wmhta2X1H72j0djn8W6rJOppmqEzvbSKb92of17+hVrX+/G37qptrZ+/dE5ta00996Xa1spqY/z+Y2/j6TeQDFEDyRA1kAxRA8kQNZAMUQPJEDWQDFEDyRA1kEyhqG0vs/287R22r616KACdmzBq232SfiHpPEmnSlph+9SqBwPQmSJH6kWSdkTEzogYlXSPpIurHQtAp4pEPUfSrkM+H2597QNsr7Q9aHvwPe0raz4AbSoS9Xhv7/rI1QojYnVEDETEwBQd0f1kADpSJOphSXMP+bxf0u5qxgHQrSJRPyPpZNsn2p4qabmkB6odC0CnJrxIQkTst32lpEck9Um6PSK2VT4ZgI4UuvJJRGyQtKHiWQCUgFeUAckQNZAMUQPJEDWQDFEDyRA1kAxRA8lUskNHnfpOOL62tYYuv7m2tSTpJ3sW1LbWkaum1baWTqvvcWnPv+pbS9KB10ZqXW88HKmBZIgaSIaogWSIGkiGqIFkiBpIhqiBZIgaSIaogWSIGkimyA4dt9sesf1sHQMB6E6RI/WvJC2reA4AJZkw6oh4StIbNcwCoASlvUvL9kpJKyVpmqaX9W0BtKm0E2VsuwP0Bs5+A8kQNZBMkV9p3S3pj5IW2B62/Z3qxwLQqSJ7aa2oYxAA5eDpN5AMUQPJEDWQDFEDyRA1kAxRA8kQNZDMpN92573PzGl6hMr8cvBLta3VP7+vtrXOu/6J2tZ66Edfrm0tSZq+nm13AJSMqIFkiBpIhqiBZIgaSIaogWSIGkiGqIFkiBpIhqiBZIpco2yu7cdtD9neZvvqOgYD0Jkir/3eL+n7EbHZ9kxJm2w/GhHPVTwbgA4U2XbnlYjY3Pp4r6QhSXnfRQFMcm29S8v2PEkLJW0c5za23QF6QOETZbaPknSvpFUR8daHb2fbHaA3FIra9hSNBb02Iu6rdiQA3Shy9tuSbpM0FBE3VD8SgG4UOVIvlnSppCW2t7b+nF/xXAA6VGTbnacluYZZAJSAV5QByRA1kAxRA8kQNZAMUQPJEDWQDFEDyRA1kMyk30vrsCe31LbWKbdeUdtakvSPy2+ub7Fl9S1179uzalvryfUfeUNhehypgWSIGkiGqIFkiBpIhqiBZIgaSIaogWSIGkiGqIFkilx4cJrtP9v+S2vbnevrGAxAZ4q8THSfpCUR8XbrUsFP234oIv5U8WwAOlDkwoMh6e3Wp1Naf6LKoQB0rujF/Ptsb5U0IunRiBh32x3bg7YH39O+kscEUFShqCPiQEScLqlf0iLbnx3nPmy7A/SAts5+R8Sbkp5QrW/UA9COIme/j7N9dOvjIyV9RdL2iucC0KEiZ79nS7rTdp/G/hH4TUQ8WO1YADpV5Oz3XzW2JzWASYBXlAHJEDWQDFEDyRA1kAxRA8kQNZAMUQPJEDWQzKTfdqdOn/riy02PkMIPnv5mbWvN12Bta/UKjtRAMkQNJEPUQDJEDSRD1EAyRA0kQ9RAMkQNJEPUQDJEDSRTOOrWBf232Oaig0APa+dIfbWkoaoGAVCOotvu9Eu6QNKaascB0K2iR+obJV0j6f2PuwN7aQG9ocgOHRdKGomITf/rfuylBfSGIkfqxZIusv2ipHskLbF9V6VTAejYhFFHxHUR0R8R8yQtl/RYRFxS+WQAOsLvqYFk2rqcUUQ8obGtbAH0KI7UQDJEDSRD1EAyRA0kQ9RAMkQNJEPUQDJsu9OGx0+7v9b1Tnz4u7WtNf0T/65trUeW3lTbWldpcW1r9QqO1EAyRA0kQ9RAMkQNJEPUQDJEDSRD1EAyRA0kQ9RAMkQNJFPoZaKtK4nulXRA0v6IGKhyKACda+e13+dExJ7KJgFQCp5+A8kUjTok/c72Jtsrx7sD2+4AvaHo0+/FEbHb9vGSHrW9PSKeOvQOEbFa0mpJmuVjouQ5ARRU6EgdEbtb/x2RtF7SoiqHAtC5IhvkzbA98+DHkr4q6dmqBwPQmSJPv0+QtN72wfv/OiIernQqAB2bMOqI2CnpczXMAqAE/EoLSIaogWSIGkiGqIFkiBpIhqiBZIgaSGbSb7vTd8LxTY9QmVN+tre2tfZ8/pja1pp/1oza1nr362fWtpYkTV+/sdb1xsORGkiGqIFkiBpIhqiBZIgaSIaogWSIGkiGqIFkiBpIhqiBZApFbfto2+tsb7c9ZPusqgcD0Jmir/2+SdLDEfEN21MlTa9wJgBdmDBq27MknS3p25IUEaOSRqsdC0Cnijz9PknS65LusL3F9prW9b8/gG13gN5QJOrDJZ0h6ZaIWCjpHUnXfvhOEbE6IgYiYmCKjih5TABFFYl6WNJwRBx8o+g6jUUOoAdNGHVEvCppl+0FrS8tlfRcpVMB6FjRs99XSVrbOvO9U9Jl1Y0EoBuFoo6IrZIGqh0FQBl4RRmQDFEDyRA1kAxRA8kQNZAMUQPJEDWQDFEDyUz6vbQOvDZS21qn3HpFbWtJ0v0bflrbWvOn1Le/VZ2mvfH/9y5hjtRAMkQNJEPUQDJEDSRD1EAyRA0kQ9RAMkQNJEPUQDITRm17ge2th/x5y/aqGmYD0IEJXyYaEc9LOl2SbPdJelnS+mrHAtCpdp9+L5X094h4qYphAHSv3Td0LJd093g32F4paaUkTWP/PKAxhY/UrWt+XyTpt+PdzrY7QG9o5+n3eZI2R8RrVQ0DoHvtRL1CH/PUG0DvKBS17emSzpV0X7XjAOhW0W133pV0bMWzACgBrygDkiFqIBmiBpIhaiAZogaSIWogGaIGkiFqIBlHRPnf1H5dUrtvz/ykpD2lD9Mbsj42HldzPh0Rx413QyVRd8L2YEQMND1HFbI+Nh5Xb+LpN5AMUQPJ9FLUq5seoEJZHxuPqwf1zM/UAMrRS0dqACUgaiCZnoja9jLbz9veYfvapucpg+25th+3PWR7m+2rm56pTLb7bG+x/WDTs5TJ9tG219ne3vq7O6vpmdrV+M/UrQ0CXtDY5ZKGJT0jaUVEPNfoYF2yPVvS7IjYbHumpE2SvjbZH9dBtr8naUDSrIi4sOl5ymL7Tkl/iIg1rSvoTo+INxseqy29cKReJGlHROyMiFFJ90i6uOGZuhYRr0TE5tbHeyUNSZrT7FTlsN0v6QJJa5qepUy2Z0k6W9JtkhQRo5MtaKk3op4jadchnw8ryf/8B9meJ2mhpI0Nj1KWGyVdI+n9huco20mSXpd0R+tHizW2ZzQ9VLt6IWqP87U0v2ezfZSkeyWtioi3mp6nW7YvlDQSEZuanqUCh0s6Q9ItEbFQ0juSJt05nl6IeljS3EM+75e0u6FZSmV7isaCXhsRWS6vvFjSRbZf1NiPSkts39XsSKUZljQcEQefUa3TWOSTSi9E/Yykk22f2DoxsVzSAw3P1DXb1tjPZkMRcUPT85QlIq6LiP6ImKexv6vHIuKShscqRUS8KmmX7QWtLy2VNOlObLa7QV7pImK/7SslPSKpT9LtEbGt4bHKsFjSpZL+Zntr62s/jIgNzY2EAq6StLZ1gNkp6bKG52lb47/SAlCuXnj6DaBERA0kQ9RAMkQNJEPUQDJEDSRD1EAy/wW5gpkXhIMfLAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import sklearn\n",
    "from sklearn import *\n",
    "X,y = sklearn.datasets.load_digits(return_X_y=True, as_frame=False)\n",
    "\n",
    "X = X.reshape(-1,8,8,1)/255\n",
    "y = y.reshape(-1).astype(int)\n",
    "Y = one_hot(y, len(np.unique(y)))\n",
    "\n",
    "X, Y, X_test, Y_test = train_test_split(X,Y,ratio=0.2)\n",
    "\n",
    "print(f\"Input size:{X.shape} Number of images:{len(X)}, Classes:{np.unique(y)}\")\n",
    "plt.imshow(X[0].reshape(8,8,1))\n",
    "print(\"number\", np.argmax(Y[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 406,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(12)\n",
    "num_iter = 20\n",
    "batch_size = 128\n",
    "\n",
    "num_batch = len(X)//batch_size\n",
    "m = num_batch*batch_size   # apply cutoff\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train feed forward neuron network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 407,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss:3.6769705365651344 Accuracy:0.0546875\n",
      "Loss:2.5056365563985197 Accuracy:0.3828125\n",
      "Loss:2.283738689819626 Accuracy:0.4375\n",
      "Loss:1.3343165965839852 Accuracy:0.671875\n",
      "Loss:0.9782908922397674 Accuracy:0.6875\n",
      "Loss:0.48455474967167694 Accuracy:0.8359375\n",
      "Loss:0.47832194834788044 Accuracy:0.8203125\n",
      "Loss:0.7947488757753864 Accuracy:0.7421875\n",
      "Loss:0.25032074905104756 Accuracy:0.9140625\n",
      "Loss:0.32235495303139117 Accuracy:0.8984375\n",
      "Loss:0.3845144847522497 Accuracy:0.9296875\n",
      "Loss:0.287412455694012 Accuracy:0.9375\n",
      "Loss:0.40121976430346357 Accuracy:0.8828125\n",
      "Loss:0.18376586715411056 Accuracy:0.9296875\n",
      "Loss:0.3002947261440367 Accuracy:0.8984375\n",
      "Loss:0.410321803584249 Accuracy:0.9140625\n",
      "Loss:0.18613047199833022 Accuracy:0.9453125\n",
      "Loss:0.2221554769405037 Accuracy:0.9296875\n",
      "Loss:0.12830442941696346 Accuracy:0.953125\n",
      "Loss:0.14383553204386992 Accuracy:0.9375\n",
      "Loss:0.20255985776061874 Accuracy:0.96875\n",
      "Loss:0.1989531170529381 Accuracy:0.9375\n",
      "Loss:0.12006859886709621 Accuracy:0.96875\n",
      "Loss:0.10794588504238267 Accuracy:0.9375\n",
      "Loss:0.08699672651477973 Accuracy:0.96875\n",
      "Loss:0.10745850545044541 Accuracy:0.9765625\n",
      "Loss:0.026212106793665663 Accuracy:0.9921875\n",
      "Loss:0.08146268334397169 Accuracy:0.9609375\n",
      "Loss:0.14278300777715186 Accuracy:0.9375\n",
      "Loss:0.06134024325329035 Accuracy:0.984375\n",
      "Loss:0.1265427385953905 Accuracy:0.9609375\n",
      "Loss:0.20152676111416973 Accuracy:0.9609375\n",
      "Loss:0.19092595823733938 Accuracy:0.9453125\n",
      "Loss:0.04547757370892766 Accuracy:0.9921875\n",
      "Loss:0.041882952681026715 Accuracy:0.9921875\n",
      "Loss:0.06334869461965878 Accuracy:0.984375\n",
      "Loss:0.048202343965064454 Accuracy:0.9765625\n",
      "Loss:0.02799586211082876 Accuracy:1.0\n",
      "Loss:0.040630885736568464 Accuracy:0.984375\n",
      "Loss:0.15585294216129886 Accuracy:0.953125\n",
      "Loss:0.028413676544619793 Accuracy:0.9921875\n",
      "Loss:0.03327047187960693 Accuracy:0.984375\n",
      "Loss:0.10480110773174867 Accuracy:0.9609375\n",
      "Loss:0.05053885624019045 Accuracy:0.984375\n",
      "Loss:0.010056432306007236 Accuracy:1.0\n",
      "Loss:0.036587137606640605 Accuracy:0.984375\n",
      "Loss:0.01801417541211331 Accuracy:1.0\n",
      "Loss:0.021703333322061636 Accuracy:1.0\n",
      "Loss:0.03960904200851077 Accuracy:0.9921875\n",
      "Loss:0.010625399636332465 Accuracy:1.0\n",
      "Loss:0.01066422309028255 Accuracy:1.0\n",
      "Loss:0.012561524966962813 Accuracy:1.0\n",
      "Loss:0.035271604609476856 Accuracy:0.984375\n",
      "Loss:0.10334690829594283 Accuracy:0.9765625\n",
      "Loss:0.05802899095577424 Accuracy:0.96875\n",
      "Loss:0.0174549527072883 Accuracy:0.9921875\n",
      "Loss:0.02496882904588135 Accuracy:0.984375\n",
      "Loss:0.06410765683410788 Accuracy:0.9765625\n",
      "Loss:0.021799785075428776 Accuracy:0.9921875\n",
      "Loss:0.014803058098980167 Accuracy:1.0\n",
      "Loss:0.012033510411222458 Accuracy:1.0\n",
      "Loss:0.045804258322131586 Accuracy:0.984375\n",
      "Loss:0.01772549884735983 Accuracy:0.9921875\n",
      "Loss:0.010546761080389662 Accuracy:1.0\n",
      "Loss:0.010326718393290763 Accuracy:1.0\n",
      "Loss:0.029104295847039385 Accuracy:0.984375\n",
      "Loss:0.022405721397698884 Accuracy:0.9921875\n",
      "Loss:0.0041730016005212655 Accuracy:1.0\n",
      "Loss:0.01033052579230405 Accuracy:0.9921875\n",
      "Loss:0.036836773197315084 Accuracy:0.984375\n",
      "Loss:0.006055560042231499 Accuracy:1.0\n",
      "Loss:0.0032931148896835336 Accuracy:1.0\n",
      "Loss:0.007329298589534679 Accuracy:1.0\n",
      "Loss:0.021031493707125853 Accuracy:1.0\n",
      "Loss:0.01045801937141075 Accuracy:1.0\n",
      "Loss:0.01958064053123536 Accuracy:1.0\n",
      "Loss:0.026013471967269372 Accuracy:0.9921875\n",
      "Loss:0.00936055385131024 Accuracy:1.0\n",
      "Loss:0.02234731544135859 Accuracy:0.9921875\n",
      "Loss:0.0064257971419912265 Accuracy:1.0\n",
      "Loss:0.027396349821153014 Accuracy:0.9921875\n",
      "Loss:0.005151079288889205 Accuracy:1.0\n",
      "Loss:0.004351815692303446 Accuracy:1.0\n",
      "Loss:0.03096123588249583 Accuracy:0.984375\n",
      "Loss:0.016829954529044035 Accuracy:0.9921875\n",
      "Loss:0.014278052270059375 Accuracy:0.9921875\n",
      "Loss:0.018694569423028842 Accuracy:0.9921875\n",
      "Loss:0.009235555203444823 Accuracy:1.0\n",
      "Loss:0.019424140510252375 Accuracy:0.9921875\n",
      "Loss:0.022059884421059404 Accuracy:0.984375\n",
      "Loss:0.003800346431038431 Accuracy:1.0\n",
      "Loss:0.005443375042622172 Accuracy:1.0\n",
      "Loss:0.013106217780737967 Accuracy:0.9921875\n",
      "Loss:0.008708332823026927 Accuracy:1.0\n",
      "Loss:0.013100326591948323 Accuracy:1.0\n",
      "Loss:0.008319868535795112 Accuracy:1.0\n",
      "Loss:0.004534085802029191 Accuracy:1.0\n",
      "Loss:0.011771407944169369 Accuracy:0.9921875\n",
      "Loss:0.005422698835690587 Accuracy:1.0\n",
      "Loss:0.009088228075503545 Accuracy:1.0\n",
      "Loss:0.006344172264202689 Accuracy:1.0\n",
      "Loss:0.020579913368237453 Accuracy:0.9921875\n",
      "Loss:0.009307337277970739 Accuracy:1.0\n",
      "Loss:0.014508843440085503 Accuracy:0.9921875\n",
      "Loss:0.02646606253291138 Accuracy:0.9921875\n",
      "Loss:0.004792277327939081 Accuracy:1.0\n",
      "Loss:0.004438543170090499 Accuracy:1.0\n",
      "Loss:0.0024721185451506487 Accuracy:1.0\n",
      "Loss:0.009634342683336971 Accuracy:1.0\n",
      "Loss:0.009509470737698004 Accuracy:0.9921875\n",
      "Loss:0.006574565350141675 Accuracy:1.0\n",
      "Loss:0.004020109526572023 Accuracy:1.0\n",
      "Loss:0.007362427576480805 Accuracy:1.0\n",
      "Loss:0.008199081411935077 Accuracy:1.0\n",
      "Loss:0.0025099514524199503 Accuracy:1.0\n",
      "Loss:0.009548103848974266 Accuracy:0.9921875\n",
      "Loss:0.0033366161000011447 Accuracy:1.0\n",
      "Loss:0.01863729634423413 Accuracy:0.9921875\n",
      "Loss:0.015851622903510335 Accuracy:0.9921875\n",
      "Loss:0.005683341026454762 Accuracy:1.0\n",
      "Loss:0.0036958462114726268 Accuracy:1.0\n",
      "Loss:0.0027075912120959117 Accuracy:1.0\n",
      "Loss:0.003444036289956184 Accuracy:1.0\n",
      "Loss:0.01040863990094546 Accuracy:0.9921875\n",
      "Loss:0.005318829484409306 Accuracy:1.0\n",
      "Loss:0.0032264822061832572 Accuracy:1.0\n",
      "Loss:0.0044471040790393865 Accuracy:1.0\n",
      "Loss:0.0036277009834759957 Accuracy:1.0\n",
      "Loss:0.0042443987194192775 Accuracy:1.0\n",
      "Loss:0.005121064978011226 Accuracy:1.0\n",
      "Loss:0.008520069832722913 Accuracy:1.0\n",
      "Loss:0.007795818105854055 Accuracy:1.0\n",
      "Loss:0.0037750063358461725 Accuracy:1.0\n",
      "Loss:0.004676468200586324 Accuracy:1.0\n",
      "Loss:0.003345211656999536 Accuracy:1.0\n",
      "Loss:0.007888738038829387 Accuracy:1.0\n",
      "Loss:0.0087121295435232 Accuracy:0.9921875\n",
      "Loss:0.004727486772852622 Accuracy:1.0\n",
      "Loss:0.003926713817961168 Accuracy:1.0\n",
      "Loss:0.003656843797542359 Accuracy:1.0\n",
      "Loss:0.0022390083570513596 Accuracy:1.0\n",
      "Loss:0.001779419124432152 Accuracy:1.0\n",
      "Loss:0.0010202713289430027 Accuracy:1.0\n",
      "Loss:0.003172105135190282 Accuracy:1.0\n",
      "Loss:0.001171077966669167 Accuracy:1.0\n",
      "Loss:0.0028176236368922802 Accuracy:1.0\n",
      "Loss:0.004404290995671829 Accuracy:1.0\n",
      "Loss:0.007178687687709538 Accuracy:1.0\n",
      "Loss:0.002743393661541938 Accuracy:1.0\n",
      "Loss:0.0014155799339911309 Accuracy:1.0\n",
      "Loss:0.005828696127829873 Accuracy:1.0\n",
      "Loss:0.002609397679598063 Accuracy:1.0\n",
      "Loss:0.003613007875925186 Accuracy:1.0\n",
      "Loss:0.005482180303133327 Accuracy:1.0\n",
      "Loss:0.0007860184189999254 Accuracy:1.0\n",
      "Loss:0.0028666684511160336 Accuracy:1.0\n",
      "Loss:0.003506922282391643 Accuracy:1.0\n",
      "Loss:0.0020352491025600636 Accuracy:1.0\n",
      "Loss:0.004111200745532436 Accuracy:1.0\n",
      "Loss:0.002616131046315405 Accuracy:1.0\n",
      "Loss:0.001969400580388209 Accuracy:1.0\n",
      "Loss:0.007558103723554703 Accuracy:1.0\n",
      "Loss:0.0014515789911546987 Accuracy:1.0\n",
      "Loss:0.007024096983612248 Accuracy:1.0\n",
      "Loss:0.0017051148383121562 Accuracy:1.0\n",
      "Loss:0.0020545336134702285 Accuracy:1.0\n",
      "Loss:0.005588885903145803 Accuracy:1.0\n",
      "Loss:0.0025669210216626753 Accuracy:1.0\n",
      "Loss:0.0061692642153439795 Accuracy:1.0\n",
      "Loss:0.0056142807215882285 Accuracy:1.0\n",
      "Loss:0.002727709865322073 Accuracy:1.0\n",
      "Loss:0.005189273079316243 Accuracy:1.0\n",
      "Loss:0.003441494185805463 Accuracy:1.0\n",
      "Loss:0.001934213550552858 Accuracy:1.0\n",
      "Loss:0.0005640683176494648 Accuracy:1.0\n",
      "Loss:0.0022973208380615232 Accuracy:1.0\n",
      "Loss:0.0015368973853892576 Accuracy:1.0\n",
      "Loss:0.0009730323431725791 Accuracy:1.0\n",
      "Loss:0.004672520239758595 Accuracy:1.0\n",
      "Loss:0.00253399357754889 Accuracy:1.0\n",
      "Loss:0.001182387470811234 Accuracy:1.0\n",
      "Loss:0.00322667567877416 Accuracy:1.0\n",
      "Loss:0.004811214367940317 Accuracy:1.0\n",
      "Loss:0.0014348593188805497 Accuracy:1.0\n",
      "Loss:0.0023860112099967244 Accuracy:1.0\n",
      "Loss:0.0014445348755174957 Accuracy:1.0\n",
      "Loss:0.000690951810619663 Accuracy:1.0\n",
      "Loss:0.0008332669845585609 Accuracy:1.0\n",
      "Loss:0.0007079680818254201 Accuracy:1.0\n",
      "Loss:0.0008610099632142601 Accuracy:1.0\n",
      "Loss:0.001288664055238272 Accuracy:1.0\n",
      "Loss:0.0008781657360434729 Accuracy:1.0\n",
      "Loss:0.001281828608545507 Accuracy:1.0\n",
      "Loss:0.0022048973316469065 Accuracy:1.0\n",
      "Loss:0.0015797105649049653 Accuracy:1.0\n",
      "Loss:0.010361199324247804 Accuracy:1.0\n",
      "Loss:0.002614389174656396 Accuracy:1.0\n",
      "Loss:0.0016995483293533145 Accuracy:1.0\n",
      "Loss:0.0019025019892262723 Accuracy:1.0\n",
      "Loss:0.0017179303359146055 Accuracy:1.0\n",
      "Loss:0.005237198192961508 Accuracy:1.0\n",
      "Loss:0.0013832435663470026 Accuracy:1.0\n",
      "Loss:0.0021969830738436526 Accuracy:1.0\n",
      "Loss:0.0009834421919757501 Accuracy:1.0\n",
      "Loss:0.0011243489808828574 Accuracy:1.0\n",
      "Loss:0.0014899595708982912 Accuracy:1.0\n",
      "Loss:0.002126850496878856 Accuracy:1.0\n",
      "Loss:0.0018809584983114904 Accuracy:1.0\n",
      "Loss:0.010561085570390365 Accuracy:1.0\n",
      "Loss:0.0025480995728729312 Accuracy:1.0\n",
      "Loss:0.0013360014916672907 Accuracy:1.0\n",
      "Loss:0.0012600760487642398 Accuracy:1.0\n",
      "Loss:0.002658865096060551 Accuracy:1.0\n",
      "Loss:0.0035810299894077826 Accuracy:1.0\n",
      "Loss:0.0004568116233742023 Accuracy:1.0\n",
      "Loss:0.002193512963981405 Accuracy:1.0\n",
      "Loss:0.0009110582980288821 Accuracy:1.0\n",
      "Loss:0.0025397078947239247 Accuracy:1.0\n",
      "Loss:0.0006174297013272514 Accuracy:1.0\n",
      "Loss:0.0025182708949347114 Accuracy:1.0\n"
     ]
    }
   ],
   "source": [
    "# Train mini batch\n",
    "np.random.seed(12)\n",
    "net = FC(X.shape[1:], out_size=Y.shape[1])\n",
    "criteria = Criteria(mode='cross_entropy')\n",
    "optimizer = Adam(net,learning_rate=0.01)\n",
    "\n",
    "\n",
    "for i in range(num_iter):\n",
    "    \n",
    "    permutation = np.random.permutation(m)\n",
    "    \n",
    "    for j in range(0,m,batch_size):\n",
    "        \n",
    "        indices = permutation[j:j+batch_size]\n",
    "        X_batch, Y_batch = X[indices], Y[indices]\n",
    "        \n",
    "        A = net.forward(X_batch)\n",
    "\n",
    "        loss = criteria.forward(A, Y_batch)\n",
    "        \n",
    "        dZ = criteria.backward()\n",
    "        dZ = net.backward(dZ)\n",
    "\n",
    "        optimizer.step(net)\n",
    "        \n",
    "        accuracy = np.sum(np.argmax(A,axis=1)==np.argmax(Y_batch,axis=1))/len(X_batch)\n",
    "        print(f\"Loss:{loss} Accuracy:{accuracy}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 408,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Num test:359 Loss:0.06828290939348204 Accuracy:0.9721448467966574\n"
     ]
    }
   ],
   "source": [
    "A = net.forward(X_test)\n",
    "loss = criteria.forward(A, Y_test)\n",
    "accuracy = np.sum(np.argmax(A,axis=1)==np.argmax(Y_test,axis=1))/len(X_test)\n",
    "\n",
    "print(f\" Num test:{len(X_test)} Loss:{loss} Accuracy:{accuracy}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "98% test accuracy! Pretty good for just 2 fully connected layers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train Convolution neuron network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 409,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss:4.314904818414068 Accuracy:0.0625\n",
      "Loss:2.298961734858465 Accuracy:0.21875\n",
      "Loss:1.8912552038124686 Accuracy:0.4453125\n",
      "Loss:1.0278178964034732 Accuracy:0.640625\n",
      "Loss:1.1861579159269475 Accuracy:0.625\n",
      "Loss:0.8227776185285441 Accuracy:0.734375\n",
      "Loss:0.7024408521403106 Accuracy:0.7578125\n",
      "Loss:0.5564132081561419 Accuracy:0.84375\n",
      "Loss:0.8195145339081696 Accuracy:0.7109375\n",
      "Loss:0.4260219679594739 Accuracy:0.8359375\n",
      "Loss:0.4821291363201202 Accuracy:0.828125\n",
      "Loss:0.5076431498882088 Accuracy:0.8671875\n",
      "Loss:0.5596296228202842 Accuracy:0.84375\n",
      "Loss:0.37937647708150807 Accuracy:0.875\n",
      "Loss:0.3309492922123606 Accuracy:0.890625\n",
      "Loss:0.3563136590438748 Accuracy:0.8984375\n",
      "Loss:0.2971427358810108 Accuracy:0.9140625\n",
      "Loss:0.502888396285432 Accuracy:0.890625\n",
      "Loss:0.2655591631625066 Accuracy:0.9375\n",
      "Loss:0.2891782007872266 Accuracy:0.9296875\n",
      "Loss:0.1671668977016157 Accuracy:0.96875\n",
      "Loss:0.20167821561756247 Accuracy:0.9453125\n",
      "Loss:0.15312721398023427 Accuracy:0.9453125\n",
      "Loss:0.12371327596131808 Accuracy:0.9453125\n",
      "Loss:0.08470559639044595 Accuracy:0.953125\n",
      "Loss:0.20762223552706283 Accuracy:0.9453125\n",
      "Loss:0.17534511709466238 Accuracy:0.9453125\n",
      "Loss:0.2206541144177602 Accuracy:0.9453125\n",
      "Loss:0.11089136927486382 Accuracy:0.9609375\n",
      "Loss:0.1314847098789365 Accuracy:0.953125\n",
      "Loss:0.12641862717101862 Accuracy:0.953125\n",
      "Loss:0.1349706270987503 Accuracy:0.953125\n",
      "Loss:0.1479113108503971 Accuracy:0.953125\n",
      "Loss:0.04497623659054042 Accuracy:0.984375\n",
      "Loss:0.10399219505853365 Accuracy:0.9609375\n",
      "Loss:0.07529436791143995 Accuracy:0.9765625\n",
      "Loss:0.06451782013655666 Accuracy:0.984375\n",
      "Loss:0.0619981706556761 Accuracy:0.984375\n",
      "Loss:0.10623482039227178 Accuracy:0.9609375\n",
      "Loss:0.044940848726764786 Accuracy:0.984375\n",
      "Loss:0.06501088196588775 Accuracy:0.9765625\n",
      "Loss:0.07627168517945129 Accuracy:0.9609375\n",
      "Loss:0.05433185078174495 Accuracy:0.9765625\n",
      "Loss:0.07448595262544488 Accuracy:0.96875\n",
      "Loss:0.031822633769720665 Accuracy:0.984375\n",
      "Loss:0.07308917129298492 Accuracy:0.9765625\n",
      "Loss:0.04118873234297639 Accuracy:0.984375\n",
      "Loss:0.03336868234081479 Accuracy:0.9921875\n",
      "Loss:0.012943119554237357 Accuracy:1.0\n",
      "Loss:0.04373927821969227 Accuracy:0.984375\n",
      "Loss:0.06686472640692114 Accuracy:0.9765625\n",
      "Loss:0.05265942019253579 Accuracy:0.9609375\n",
      "Loss:0.029975388363833056 Accuracy:0.9765625\n",
      "Loss:0.03418784345947186 Accuracy:0.984375\n",
      "Loss:0.07215463353140476 Accuracy:0.984375\n",
      "Loss:0.020486035337805672 Accuracy:0.9921875\n",
      "Loss:0.045508251515326864 Accuracy:0.96875\n",
      "Loss:0.07152100192985078 Accuracy:0.96875\n",
      "Loss:0.05299255896682591 Accuracy:0.984375\n",
      "Loss:0.012312187924117052 Accuracy:1.0\n",
      "Loss:0.016905826938957768 Accuracy:1.0\n",
      "Loss:0.04417520304370301 Accuracy:0.9765625\n",
      "Loss:0.05872490617605551 Accuracy:0.9765625\n",
      "Loss:0.09909016953422844 Accuracy:0.953125\n",
      "Loss:0.047026343263686265 Accuracy:0.984375\n",
      "Loss:0.04258163387551626 Accuracy:0.9921875\n",
      "Loss:0.028629633203344086 Accuracy:0.9921875\n",
      "Loss:0.014397027243860876 Accuracy:1.0\n",
      "Loss:0.0028598351326242642 Accuracy:1.0\n",
      "Loss:0.0054817403686487146 Accuracy:1.0\n",
      "Loss:0.027235418708077393 Accuracy:0.9921875\n",
      "Loss:0.04190117891904605 Accuracy:0.984375\n",
      "Loss:0.07410526947973983 Accuracy:0.9765625\n",
      "Loss:0.03269671968244035 Accuracy:0.984375\n",
      "Loss:0.026229722917338313 Accuracy:0.9921875\n",
      "Loss:0.013195983863949998 Accuracy:1.0\n",
      "Loss:0.023051951219582624 Accuracy:0.9921875\n",
      "Loss:0.017408033272653844 Accuracy:1.0\n",
      "Loss:0.0385184234764384 Accuracy:0.984375\n",
      "Loss:0.02393243344188888 Accuracy:0.9921875\n",
      "Loss:0.015183280539265334 Accuracy:0.9921875\n",
      "Loss:0.02539644968121229 Accuracy:0.9921875\n",
      "Loss:0.009219472552451109 Accuracy:1.0\n",
      "Loss:0.007094358696702663 Accuracy:1.0\n",
      "Loss:0.01929177113067619 Accuracy:0.9921875\n",
      "Loss:0.04215870972124455 Accuracy:0.9765625\n",
      "Loss:0.004005586098484707 Accuracy:1.0\n",
      "Loss:0.006465490925667075 Accuracy:1.0\n",
      "Loss:0.03589887630207677 Accuracy:0.9921875\n",
      "Loss:0.014805536026890413 Accuracy:0.9921875\n",
      "Loss:0.0033936324174572287 Accuracy:1.0\n",
      "Loss:0.004217836675536735 Accuracy:1.0\n",
      "Loss:0.010534068465596715 Accuracy:1.0\n",
      "Loss:0.032877769276899324 Accuracy:0.984375\n",
      "Loss:0.008842767708268405 Accuracy:1.0\n",
      "Loss:0.01082316685941541 Accuracy:0.9921875\n",
      "Loss:0.010602196159621845 Accuracy:1.0\n",
      "Loss:0.01348688615104119 Accuracy:0.9921875\n",
      "Loss:0.03157942837648971 Accuracy:0.984375\n",
      "Loss:0.011572596882630397 Accuracy:1.0\n",
      "Loss:0.004483614786301068 Accuracy:1.0\n",
      "Loss:0.01188078641145466 Accuracy:0.9921875\n",
      "Loss:0.005105456246101334 Accuracy:1.0\n",
      "Loss:0.011905448362507965 Accuracy:1.0\n",
      "Loss:0.009255570662168078 Accuracy:1.0\n",
      "Loss:0.012622835300286817 Accuracy:0.9921875\n",
      "Loss:0.006844429213043537 Accuracy:1.0\n",
      "Loss:0.009829189980725288 Accuracy:1.0\n",
      "Loss:0.0374749847083167 Accuracy:0.984375\n",
      "Loss:0.010892159253099134 Accuracy:0.9921875\n",
      "Loss:0.0031172196836846367 Accuracy:1.0\n",
      "Loss:0.0024386442937745613 Accuracy:1.0\n",
      "Loss:0.003783978729396888 Accuracy:1.0\n",
      "Loss:0.008604801621507802 Accuracy:0.9921875\n",
      "Loss:0.01174247382671696 Accuracy:1.0\n",
      "Loss:0.005023796506071989 Accuracy:1.0\n",
      "Loss:0.013932347881718212 Accuracy:1.0\n",
      "Loss:0.00801674166608713 Accuracy:1.0\n",
      "Loss:0.0033969777109722913 Accuracy:1.0\n",
      "Loss:0.004027582942285381 Accuracy:1.0\n",
      "Loss:0.00825769848318756 Accuracy:1.0\n",
      "Loss:0.013401007154260333 Accuracy:0.9921875\n",
      "Loss:0.0023902307216034646 Accuracy:1.0\n",
      "Loss:0.009362443300154209 Accuracy:1.0\n",
      "Loss:0.005595023652640448 Accuracy:1.0\n",
      "Loss:0.0015793150879482194 Accuracy:1.0\n",
      "Loss:0.00974714542037805 Accuracy:1.0\n",
      "Loss:0.011053924746310718 Accuracy:1.0\n",
      "Loss:0.00811276729787877 Accuracy:1.0\n",
      "Loss:0.003241744328425623 Accuracy:1.0\n",
      "Loss:0.012708157941480345 Accuracy:0.9921875\n",
      "Loss:0.007392594767402486 Accuracy:1.0\n",
      "Loss:0.006055529379443475 Accuracy:1.0\n",
      "Loss:0.0018690627788571078 Accuracy:1.0\n",
      "Loss:0.004430221195939638 Accuracy:1.0\n",
      "Loss:0.005007721081414326 Accuracy:1.0\n",
      "Loss:0.01768342183060427 Accuracy:0.9921875\n",
      "Loss:0.0044836459594852565 Accuracy:1.0\n",
      "Loss:0.003803675337396257 Accuracy:1.0\n",
      "Loss:0.0016283723626331244 Accuracy:1.0\n",
      "Loss:0.005873992255856475 Accuracy:1.0\n",
      "Loss:0.0012688782499696073 Accuracy:1.0\n",
      "Loss:0.007969866494644144 Accuracy:1.0\n",
      "Loss:0.006166499200943296 Accuracy:1.0\n",
      "Loss:0.0030980053163233425 Accuracy:1.0\n",
      "Loss:0.01607970205165584 Accuracy:0.9921875\n",
      "Loss:0.004248132907599814 Accuracy:1.0\n",
      "Loss:0.005835024185737223 Accuracy:1.0\n",
      "Loss:0.0033678791243563645 Accuracy:1.0\n",
      "Loss:0.001965116226230469 Accuracy:1.0\n",
      "Loss:0.011720688722013081 Accuracy:1.0\n",
      "Loss:0.003544732417785128 Accuracy:1.0\n",
      "Loss:0.008626955279371106 Accuracy:0.9921875\n",
      "Loss:0.004642607785373048 Accuracy:1.0\n",
      "Loss:0.006126021159705817 Accuracy:1.0\n",
      "Loss:0.0015260334902825684 Accuracy:1.0\n",
      "Loss:0.0023692247235956206 Accuracy:1.0\n",
      "Loss:0.0007610124431511752 Accuracy:1.0\n",
      "Loss:0.0030345349003169223 Accuracy:1.0\n",
      "Loss:0.004891860676530552 Accuracy:1.0\n",
      "Loss:0.0014983436949738023 Accuracy:1.0\n",
      "Loss:0.005485660162699057 Accuracy:1.0\n",
      "Loss:0.009862435254318899 Accuracy:0.9921875\n",
      "Loss:0.004209522340717787 Accuracy:1.0\n",
      "Loss:0.0019553305039576798 Accuracy:1.0\n",
      "Loss:0.0039335167898601935 Accuracy:1.0\n",
      "Loss:0.002227398167083284 Accuracy:1.0\n",
      "Loss:0.00750106355361502 Accuracy:1.0\n",
      "Loss:0.013212408113809242 Accuracy:0.9921875\n",
      "Loss:0.010411796855530231 Accuracy:1.0\n",
      "Loss:0.005061355761574892 Accuracy:1.0\n",
      "Loss:0.0015600614472079002 Accuracy:1.0\n",
      "Loss:0.0016944248703042635 Accuracy:1.0\n",
      "Loss:0.004243110437070685 Accuracy:1.0\n",
      "Loss:0.004382358255759184 Accuracy:1.0\n",
      "Loss:0.004034948745175776 Accuracy:1.0\n",
      "Loss:0.008353117759418823 Accuracy:1.0\n",
      "Loss:0.0024350608751134975 Accuracy:1.0\n",
      "Loss:0.0024400947655603693 Accuracy:1.0\n",
      "Loss:0.0018285575376706896 Accuracy:1.0\n",
      "Loss:0.0014214960027742968 Accuracy:1.0\n",
      "Loss:0.0023683836779249744 Accuracy:1.0\n",
      "Loss:0.0024952427598625435 Accuracy:1.0\n",
      "Loss:0.0022390264117767593 Accuracy:1.0\n",
      "Loss:0.005333754986945138 Accuracy:1.0\n",
      "Loss:0.0012036603965583342 Accuracy:1.0\n",
      "Loss:0.0033474180603326725 Accuracy:1.0\n",
      "Loss:0.004143067688025347 Accuracy:1.0\n",
      "Loss:0.0011896973363537287 Accuracy:1.0\n",
      "Loss:0.0010704641767802557 Accuracy:1.0\n",
      "Loss:0.0028566502421197796 Accuracy:1.0\n",
      "Loss:0.002226315092118832 Accuracy:1.0\n",
      "Loss:0.0022465923846246467 Accuracy:1.0\n",
      "Loss:0.0034800510213264476 Accuracy:1.0\n",
      "Loss:0.0023360905571033186 Accuracy:1.0\n",
      "Loss:0.0007460709257689816 Accuracy:1.0\n",
      "Loss:0.010730282304038344 Accuracy:1.0\n",
      "Loss:0.005800940914048893 Accuracy:1.0\n",
      "Loss:0.01182498759491261 Accuracy:0.9921875\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss:0.0012669867723736805 Accuracy:1.0\n",
      "Loss:0.001651421408074517 Accuracy:1.0\n",
      "Loss:0.001364423161500163 Accuracy:1.0\n",
      "Loss:0.0015944429909627487 Accuracy:1.0\n",
      "Loss:0.0029154046572250555 Accuracy:1.0\n",
      "Loss:0.0010684686605284361 Accuracy:1.0\n",
      "Loss:0.0017439528672491904 Accuracy:1.0\n",
      "Loss:0.0024773945002091657 Accuracy:1.0\n",
      "Loss:0.005072311602157106 Accuracy:1.0\n",
      "Loss:0.011481203820392716 Accuracy:0.9921875\n",
      "Loss:0.003767143542558842 Accuracy:1.0\n",
      "Loss:0.003851576333087927 Accuracy:1.0\n",
      "Loss:0.0019096752247304554 Accuracy:1.0\n",
      "Loss:0.0014717846436220886 Accuracy:1.0\n",
      "Loss:0.002537646744027946 Accuracy:1.0\n",
      "Loss:0.0009436660915379267 Accuracy:1.0\n",
      "Loss:0.0006962235895101304 Accuracy:1.0\n",
      "Loss:0.0015023168465423072 Accuracy:1.0\n",
      "Loss:0.004481619366005525 Accuracy:1.0\n",
      "Loss:0.0024828597923188447 Accuracy:1.0\n",
      "Loss:0.0023031588543201546 Accuracy:1.0\n"
     ]
    }
   ],
   "source": [
    "# Train mini batch\n",
    "np.random.seed(12)\n",
    "\n",
    "\n",
    "net = CNN(in_shape=X.shape[1:], out_size=Y.shape[1])\n",
    "criteria = Criteria(mode='cross_entropy')\n",
    "optimizer = Adam(net,learning_rate=0.01)\n",
    "\n",
    "\n",
    "for i in range(num_iter):\n",
    "    \n",
    "    permutation = np.random.permutation(m)\n",
    "    \n",
    "    for j in range(0,m,batch_size):\n",
    "        \n",
    "        indices = permutation[j:j+batch_size]\n",
    "        X_batch, Y_batch = X[indices], Y[indices]\n",
    "        \n",
    "        A = net.forward(X_batch)\n",
    "\n",
    "        loss = criteria.forward(A, Y_batch)\n",
    "        \n",
    "        dZ = criteria.backward()\n",
    "        dZ = net.backward(dZ)\n",
    "\n",
    "        optimizer.step(net)\n",
    "        \n",
    "        accuracy = np.sum(np.argmax(A,axis=1)==np.argmax(Y_batch,axis=1))/len(X_batch)\n",
    "        print(f\"Loss:{loss} Accuracy:{accuracy}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 411,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Num test:359 Loss:0.0958732188884218 Accuracy:0.9721448467966574\n"
     ]
    }
   ],
   "source": [
    "A = net.forward(X_test)\n",
    "loss = criteria.forward(A, Y_test)\n",
    "accuracy = np.sum(np.argmax(A,axis=1)==np.argmax(Y_test,axis=1))/len(X_test)\n",
    "\n",
    "print(f\" Num test:{len(X_test)} Loss:{loss} Accuracy:{accuracy}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We doesn't doing any better here, asking about why? Maybe data is too simple."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiment on CIFAR10 dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import sklearn\n",
    "from sklearn import *\n",
    "\n",
    "def rgb2gray(rgb):\n",
    "    return np.dot(rgb[...,:3], [0.2989, 0.5870, 0.1140])\n",
    "\n",
    "try:\n",
    "    with open('cifar10.pickle', 'rb') as handle:\n",
    "        data = pickle.load(handle)\n",
    "except FileNotFoundError:\n",
    "    data = sklearn.datasets.fetch_openml('CIFAR_10',cache=True)\n",
    "    with open('cifar10.pickle', 'wb') as handle:\n",
    "        pickle.dump(data,handle)\n",
    "\n",
    "X = data['data'].reshape(-1,3,32,32).swapaxes(1,-1)/255\n",
    "\n",
    "y = data['target'].reshape(-1).astype(int)\n",
    "Y = one_hot(y, len(np.unique(y)))\n",
    "\n",
    "X, Y, X_test, Y_test = train_test_split(X,Y,ratio=0.1)\n",
    "\n",
    "print(f\"Input size:{X.shape} Number of images:{len(X)}, Classes:{np.unique(y)}\")\n",
    "plt.imshow(X[0])\n",
    "print(Y[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(12)\n",
    "num_iter = 2\n",
    "batch_size = 128\n",
    "\n",
    "num_batch = len(X)//batch_size\n",
    "m = num_batch*batch_size   # apply cutoff\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fully-connected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss:3.751997473970731 Accuracy:0.0859375\n",
      "Loss:12.616937309094272 Accuracy:0.125\n",
      "Loss:13.741365353183923 Accuracy:0.1875\n",
      "Loss:15.312996141882644 Accuracy:0.15625\n",
      "Loss:15.817872415559366 Accuracy:0.125\n",
      "Loss:16.39006279302169 Accuracy:0.1015625\n",
      "Loss:17.601681540566453 Accuracy:0.0390625\n",
      "Loss:15.462143711006584 Accuracy:0.1484375\n",
      "Loss:15.257817402021848 Accuracy:0.1484375\n",
      "Loss:16.01117436472375 Accuracy:0.109375\n",
      "Loss:14.51514030198485 Accuracy:0.1640625\n",
      "Loss:14.362351605045415 Accuracy:0.1875\n",
      "Loss:14.256815677702424 Accuracy:0.2109375\n",
      "Loss:13.731306741232647 Accuracy:0.203125\n",
      "Loss:13.58015492097518 Accuracy:0.203125\n",
      "Loss:14.32004255983972 Accuracy:0.15625\n",
      "Loss:12.952291057107562 Accuracy:0.171875\n",
      "Loss:14.17936065750945 Accuracy:0.125\n",
      "Loss:13.885488132687545 Accuracy:0.1484375\n",
      "Loss:13.682615991909302 Accuracy:0.15625\n",
      "Loss:13.371668951832705 Accuracy:0.15625\n",
      "Loss:10.66903513857437 Accuracy:0.2421875\n",
      "Loss:12.410989132865582 Accuracy:0.1953125\n",
      "Loss:10.96640542189833 Accuracy:0.203125\n",
      "Loss:10.881998039745882 Accuracy:0.25\n",
      "Loss:10.692964365311946 Accuracy:0.234375\n",
      "Loss:10.293431614459283 Accuracy:0.2265625\n",
      "Loss:11.017250126650413 Accuracy:0.234375\n",
      "Loss:9.901168479172647 Accuracy:0.171875\n",
      "Loss:9.97244019737876 Accuracy:0.1875\n",
      "Loss:9.589089723972094 Accuracy:0.1640625\n",
      "Loss:9.381936583541787 Accuracy:0.2421875\n",
      "Loss:8.980999876256798 Accuracy:0.234375\n",
      "Loss:8.637749634127573 Accuracy:0.2265625\n",
      "Loss:7.978982366443172 Accuracy:0.2109375\n",
      "Loss:7.686303217518942 Accuracy:0.234375\n",
      "Loss:6.613049100649443 Accuracy:0.234375\n",
      "Loss:8.62876684124907 Accuracy:0.1875\n",
      "Loss:6.406080354966714 Accuracy:0.2734375\n",
      "Loss:8.451638264192432 Accuracy:0.1640625\n",
      "Loss:7.1413228919985645 Accuracy:0.265625\n",
      "Loss:7.544419137061805 Accuracy:0.2109375\n",
      "Loss:6.259259325648314 Accuracy:0.2890625\n",
      "Loss:6.66975853420644 Accuracy:0.2265625\n",
      "Loss:6.7295645730048115 Accuracy:0.1484375\n",
      "Loss:6.60392371937081 Accuracy:0.2109375\n",
      "Loss:5.912482183448446 Accuracy:0.234375\n",
      "Loss:6.2289124599674865 Accuracy:0.1953125\n",
      "Loss:4.765605209589246 Accuracy:0.3046875\n",
      "Loss:5.262898788394512 Accuracy:0.265625\n",
      "Loss:4.168521100782631 Accuracy:0.3203125\n",
      "Loss:3.9990312319055157 Accuracy:0.3125\n",
      "Loss:4.2563396537895235 Accuracy:0.2890625\n",
      "Loss:5.5659202069881 Accuracy:0.1875\n",
      "Loss:5.644326060826781 Accuracy:0.2265625\n",
      "Loss:5.002460743348348 Accuracy:0.25\n",
      "Loss:4.748816331183175 Accuracy:0.2734375\n",
      "Loss:4.916636210995021 Accuracy:0.2421875\n",
      "Loss:4.891936108535402 Accuracy:0.21875\n",
      "Loss:5.060127571913573 Accuracy:0.234375\n",
      "Loss:4.795209581960927 Accuracy:0.2421875\n",
      "Loss:4.560390157572231 Accuracy:0.2265625\n",
      "Loss:4.710538144343293 Accuracy:0.21875\n",
      "Loss:4.024319570172208 Accuracy:0.2578125\n",
      "Loss:4.569335899747223 Accuracy:0.25\n",
      "Loss:4.24562614899794 Accuracy:0.234375\n",
      "Loss:3.625238393029734 Accuracy:0.203125\n",
      "Loss:3.683860752681158 Accuracy:0.21875\n",
      "Loss:3.721800869958318 Accuracy:0.2890625\n",
      "Loss:4.0439634725974685 Accuracy:0.2109375\n",
      "Loss:4.4182306692445446 Accuracy:0.2421875\n",
      "Loss:3.3236965101519 Accuracy:0.3203125\n",
      "Loss:3.251924067923982 Accuracy:0.2265625\n",
      "Loss:3.8294459448117126 Accuracy:0.234375\n",
      "Loss:2.8391883841442396 Accuracy:0.2890625\n",
      "Loss:3.1622192667561047 Accuracy:0.2578125\n",
      "Loss:3.3768934758913822 Accuracy:0.1953125\n",
      "Loss:2.8662646189586014 Accuracy:0.28125\n",
      "Loss:3.0660867385095285 Accuracy:0.2734375\n",
      "Loss:2.632781944402147 Accuracy:0.3046875\n",
      "Loss:3.5377333833071356 Accuracy:0.2265625\n",
      "Loss:3.330034416263749 Accuracy:0.25\n",
      "Loss:2.7694094003364595 Accuracy:0.296875\n",
      "Loss:3.8812795638124733 Accuracy:0.234375\n",
      "Loss:4.083432776714561 Accuracy:0.328125\n",
      "Loss:4.188265620011895 Accuracy:0.3046875\n",
      "Loss:3.190055842545723 Accuracy:0.265625\n",
      "Loss:3.2933110575773554 Accuracy:0.2421875\n",
      "Loss:3.1636457429453353 Accuracy:0.21875\n",
      "Loss:3.437395223625486 Accuracy:0.2109375\n",
      "Loss:4.0790573535635275 Accuracy:0.25\n",
      "Loss:3.2217572992433263 Accuracy:0.2421875\n",
      "Loss:2.7993563003057744 Accuracy:0.34375\n",
      "Loss:3.0804051006845796 Accuracy:0.3203125\n",
      "Loss:3.4522242565975905 Accuracy:0.28125\n",
      "Loss:2.819884603452149 Accuracy:0.3515625\n",
      "Loss:2.821512480855108 Accuracy:0.3046875\n",
      "Loss:3.108037866842581 Accuracy:0.296875\n",
      "Loss:3.191577136483129 Accuracy:0.2734375\n",
      "Loss:2.8970633869095943 Accuracy:0.2578125\n",
      "Loss:2.614782538130285 Accuracy:0.2890625\n",
      "Loss:2.431311802266405 Accuracy:0.296875\n",
      "Loss:2.3472019762641514 Accuracy:0.40625\n",
      "Loss:2.481248319896981 Accuracy:0.3359375\n",
      "Loss:2.689803439775292 Accuracy:0.2421875\n",
      "Loss:2.9847897587866195 Accuracy:0.265625\n",
      "Loss:2.9856237319683805 Accuracy:0.2890625\n",
      "Loss:2.8669006743183036 Accuracy:0.2734375\n",
      "Loss:2.7524568350713254 Accuracy:0.296875\n",
      "Loss:2.942967867437173 Accuracy:0.28125\n",
      "Loss:2.821793098276358 Accuracy:0.21875\n",
      "Loss:2.430182323393626 Accuracy:0.2890625\n",
      "Loss:2.595012027921178 Accuracy:0.328125\n",
      "Loss:3.3608220751662357 Accuracy:0.25\n",
      "Loss:2.8548621516218864 Accuracy:0.2578125\n",
      "Loss:2.6234224217217985 Accuracy:0.3046875\n",
      "Loss:3.070892002154293 Accuracy:0.234375\n",
      "Loss:2.7879146160770993 Accuracy:0.3125\n",
      "Loss:3.234466528842092 Accuracy:0.25\n",
      "Loss:2.8016816198250236 Accuracy:0.296875\n",
      "Loss:2.6931864119071394 Accuracy:0.3359375\n",
      "Loss:2.452372768752095 Accuracy:0.3828125\n",
      "Loss:2.9795098559286233 Accuracy:0.3515625\n",
      "Loss:2.9789077017808063 Accuracy:0.3125\n",
      "Loss:2.750911493777809 Accuracy:0.2109375\n",
      "Loss:2.4889817648504087 Accuracy:0.2890625\n",
      "Loss:3.278075424526712 Accuracy:0.25\n",
      "Loss:2.859093580168058 Accuracy:0.265625\n",
      "Loss:2.5271245684780217 Accuracy:0.34375\n",
      "Loss:3.2407207841557204 Accuracy:0.3203125\n",
      "Loss:2.776227448516077 Accuracy:0.28125\n",
      "Loss:2.5357789090153044 Accuracy:0.2109375\n",
      "Loss:2.5232977010641062 Accuracy:0.2109375\n",
      "Loss:3.0384591056049044 Accuracy:0.2421875\n",
      "Loss:3.2763909781788323 Accuracy:0.1640625\n",
      "Loss:3.0505634270025794 Accuracy:0.2734375\n",
      "Loss:3.083317919980293 Accuracy:0.234375\n",
      "Loss:3.206402393203102 Accuracy:0.328125\n",
      "Loss:2.9371153628347546 Accuracy:0.3046875\n",
      "Loss:2.838582855002353 Accuracy:0.3046875\n",
      "Loss:2.661409485953839 Accuracy:0.3515625\n",
      "Loss:2.5976364014710107 Accuracy:0.3046875\n",
      "Loss:2.920421944403947 Accuracy:0.2265625\n",
      "Loss:2.4867708949731906 Accuracy:0.2578125\n",
      "Loss:2.2713809651952537 Accuracy:0.3203125\n",
      "Loss:2.5469129343442334 Accuracy:0.3359375\n",
      "Loss:3.1987249886871596 Accuracy:0.25\n",
      "Loss:2.552071006835658 Accuracy:0.3359375\n",
      "Loss:2.055516424979591 Accuracy:0.34375\n",
      "Loss:2.7061476077197484 Accuracy:0.28125\n",
      "Loss:2.422690041211486 Accuracy:0.3359375\n",
      "Loss:2.5105475593608473 Accuracy:0.2890625\n",
      "Loss:2.3194368279510664 Accuracy:0.3203125\n",
      "Loss:2.549121267113067 Accuracy:0.28125\n",
      "Loss:2.285699967469368 Accuracy:0.34375\n",
      "Loss:2.3619826414158682 Accuracy:0.3984375\n",
      "Loss:2.2419153475226223 Accuracy:0.3359375\n",
      "Loss:2.5526373010696095 Accuracy:0.2109375\n",
      "Loss:2.134293916989843 Accuracy:0.2890625\n",
      "Loss:2.11568994586265 Accuracy:0.40625\n",
      "Loss:1.8784187348044186 Accuracy:0.34375\n",
      "Loss:2.31993402333162 Accuracy:0.203125\n",
      "Loss:2.116499483065547 Accuracy:0.3046875\n",
      "Loss:2.4313849025989764 Accuracy:0.375\n",
      "Loss:1.9063004026655475 Accuracy:0.3984375\n",
      "Loss:2.1872359145094533 Accuracy:0.2734375\n",
      "Loss:2.193004962943205 Accuracy:0.3359375\n",
      "Loss:2.4234926320336143 Accuracy:0.2734375\n",
      "Loss:2.0377121901488637 Accuracy:0.34375\n",
      "Loss:1.806532981486459 Accuracy:0.390625\n",
      "Loss:2.093172894197865 Accuracy:0.3125\n",
      "Loss:1.767918177747596 Accuracy:0.40625\n",
      "Loss:2.0826509917796105 Accuracy:0.3671875\n",
      "Loss:2.2516650937286347 Accuracy:0.328125\n",
      "Loss:2.4475943941563543 Accuracy:0.3125\n",
      "Loss:2.352249645158396 Accuracy:0.3203125\n",
      "Loss:2.2747190877084433 Accuracy:0.3203125\n",
      "Loss:2.223745914900184 Accuracy:0.3203125\n",
      "Loss:2.6700295379424004 Accuracy:0.2421875\n",
      "Loss:1.948935660950808 Accuracy:0.3359375\n",
      "Loss:2.1359785074782094 Accuracy:0.3046875\n",
      "Loss:2.2291908382204 Accuracy:0.3359375\n",
      "Loss:2.2736184525287975 Accuracy:0.3515625\n",
      "Loss:2.163867184049421 Accuracy:0.3671875\n",
      "Loss:2.227370010659776 Accuracy:0.3203125\n",
      "Loss:1.7827047664837583 Accuracy:0.40625\n",
      "Loss:2.103698631859163 Accuracy:0.2734375\n",
      "Loss:1.9664460958264423 Accuracy:0.359375\n",
      "Loss:1.9571994048641628 Accuracy:0.3515625\n",
      "Loss:1.8823141349657742 Accuracy:0.359375\n",
      "Loss:2.119483089095805 Accuracy:0.359375\n",
      "Loss:2.169505158416575 Accuracy:0.2890625\n",
      "Loss:2.1225682439636677 Accuracy:0.328125\n",
      "Loss:2.237755451671559 Accuracy:0.265625\n",
      "Loss:2.3197924284874705 Accuracy:0.2734375\n",
      "Loss:2.1672821008651715 Accuracy:0.328125\n",
      "Loss:2.4531232062411306 Accuracy:0.2265625\n",
      "Loss:2.2522632618007634 Accuracy:0.3125\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss:2.1382414047316884 Accuracy:0.3359375\n",
      "Loss:2.1730104376735344 Accuracy:0.2890625\n",
      "Loss:2.124529748110164 Accuracy:0.28125\n",
      "Loss:1.9773997551029996 Accuracy:0.3359375\n",
      "Loss:1.9835283863772704 Accuracy:0.3203125\n",
      "Loss:2.095915910889188 Accuracy:0.3515625\n",
      "Loss:2.25314594956015 Accuracy:0.2265625\n",
      "Loss:2.101594380910267 Accuracy:0.2890625\n",
      "Loss:2.1122023282683826 Accuracy:0.328125\n",
      "Loss:1.9853396524751035 Accuracy:0.359375\n",
      "Loss:2.3825232417478306 Accuracy:0.2734375\n",
      "Loss:2.0017119776709755 Accuracy:0.328125\n",
      "Loss:2.0365661162180775 Accuracy:0.328125\n",
      "Loss:2.296858683006739 Accuracy:0.3359375\n",
      "Loss:2.154525493525824 Accuracy:0.3125\n",
      "Loss:1.9076898262171957 Accuracy:0.3359375\n",
      "Loss:2.274648430571532 Accuracy:0.234375\n",
      "Loss:2.09521199874223 Accuracy:0.3203125\n",
      "Loss:1.9041275400584958 Accuracy:0.3984375\n",
      "Loss:2.0312282196996456 Accuracy:0.28125\n",
      "Loss:2.1107558884314708 Accuracy:0.359375\n",
      "Loss:1.9374265046474357 Accuracy:0.3671875\n",
      "Loss:2.075286018537714 Accuracy:0.34375\n",
      "Loss:2.0983477936931627 Accuracy:0.34375\n",
      "Loss:1.9083495481428345 Accuracy:0.3828125\n",
      "Loss:2.33206594996901 Accuracy:0.28125\n",
      "Loss:1.9677164569768613 Accuracy:0.3984375\n",
      "Loss:1.9567016066178957 Accuracy:0.3125\n",
      "Loss:1.963427324647709 Accuracy:0.3046875\n",
      "Loss:1.8378250708663655 Accuracy:0.3984375\n",
      "Loss:1.877668452050812 Accuracy:0.2890625\n",
      "Loss:1.971090469924265 Accuracy:0.28125\n",
      "Loss:2.0391349390606974 Accuracy:0.28125\n",
      "Loss:1.9979573232876573 Accuracy:0.2734375\n",
      "Loss:1.697147372666828 Accuracy:0.4453125\n",
      "Loss:1.7936578462431956 Accuracy:0.375\n",
      "Loss:2.0015164597467887 Accuracy:0.34375\n",
      "Loss:2.0087361767975764 Accuracy:0.34375\n",
      "Loss:2.0141215203009275 Accuracy:0.3203125\n",
      "Loss:2.1897752559323074 Accuracy:0.3359375\n",
      "Loss:1.7602618940411772 Accuracy:0.4453125\n",
      "Loss:1.8117640985079653 Accuracy:0.375\n",
      "Loss:1.7532012047195875 Accuracy:0.4296875\n",
      "Loss:1.7898070772132875 Accuracy:0.328125\n",
      "Loss:1.8622508182554822 Accuracy:0.390625\n",
      "Loss:2.0014990534936716 Accuracy:0.25\n",
      "Loss:1.9913084750063679 Accuracy:0.296875\n",
      "Loss:1.7918484757394109 Accuracy:0.3671875\n",
      "Loss:2.1073016535661058 Accuracy:0.3515625\n",
      "Loss:2.052502641957803 Accuracy:0.3359375\n",
      "Loss:2.1529459183989017 Accuracy:0.328125\n",
      "Loss:2.483908316254203 Accuracy:0.34375\n",
      "Loss:2.319860213685491 Accuracy:0.34375\n",
      "Loss:2.0617887702232016 Accuracy:0.3828125\n",
      "Loss:1.9644092625903062 Accuracy:0.359375\n",
      "Loss:2.165982726978119 Accuracy:0.375\n",
      "Loss:2.311850042749472 Accuracy:0.3125\n",
      "Loss:1.9640329490960406 Accuracy:0.3125\n",
      "Loss:2.150497940449018 Accuracy:0.328125\n",
      "Loss:1.8768150776880295 Accuracy:0.375\n",
      "Loss:1.8907009503206078 Accuracy:0.3203125\n",
      "Loss:1.7276746196263992 Accuracy:0.4453125\n",
      "Loss:1.935461919539161 Accuracy:0.375\n",
      "Loss:2.073588979608654 Accuracy:0.3671875\n",
      "Loss:1.9916266273131777 Accuracy:0.296875\n",
      "Loss:1.946743484457197 Accuracy:0.375\n",
      "Loss:1.931122812576749 Accuracy:0.3984375\n",
      "Loss:2.005288409975317 Accuracy:0.3515625\n",
      "Loss:1.862430629785205 Accuracy:0.3125\n",
      "Loss:1.949200620645795 Accuracy:0.3515625\n",
      "Loss:2.1013345214525887 Accuracy:0.3515625\n",
      "Loss:2.095226840194531 Accuracy:0.3359375\n",
      "Loss:2.0346761577497867 Accuracy:0.328125\n",
      "Loss:1.7692834319001616 Accuracy:0.3671875\n",
      "Loss:1.8828158027651476 Accuracy:0.3125\n",
      "Loss:1.9448438282298306 Accuracy:0.328125\n",
      "Loss:1.7721435449606537 Accuracy:0.3828125\n",
      "Loss:1.717197371896364 Accuracy:0.3671875\n",
      "Loss:1.718832439594605 Accuracy:0.4296875\n",
      "Loss:1.9108795511873247 Accuracy:0.390625\n",
      "Loss:1.9444684842802233 Accuracy:0.4140625\n",
      "Loss:1.9723470995970676 Accuracy:0.34375\n",
      "Loss:1.7442139810867685 Accuracy:0.40625\n",
      "Loss:1.9447812207034418 Accuracy:0.3359375\n",
      "Loss:1.9117150668357392 Accuracy:0.375\n",
      "Loss:2.3788347642260845 Accuracy:0.28125\n",
      "Loss:1.9386060933419347 Accuracy:0.40625\n",
      "Loss:2.0558575382564586 Accuracy:0.3515625\n",
      "Loss:2.1053105270968637 Accuracy:0.3671875\n",
      "Loss:2.3758962251882663 Accuracy:0.3203125\n",
      "Loss:2.1250096337094897 Accuracy:0.296875\n",
      "Loss:2.2486200600361865 Accuracy:0.3671875\n",
      "Loss:2.166396168197065 Accuracy:0.265625\n",
      "Loss:1.7122614610039095 Accuracy:0.421875\n",
      "Loss:1.9692609243528372 Accuracy:0.3046875\n",
      "Loss:2.0935838999896204 Accuracy:0.359375\n",
      "Loss:2.1814803988570137 Accuracy:0.2890625\n",
      "Loss:1.8582121937270455 Accuracy:0.3125\n",
      "Loss:1.9196316164437237 Accuracy:0.359375\n",
      "Loss:2.036762037631686 Accuracy:0.3359375\n",
      "Loss:2.034826424070512 Accuracy:0.3125\n",
      "Loss:1.786364183958125 Accuracy:0.3984375\n",
      "Loss:1.828761481402802 Accuracy:0.40625\n",
      "Loss:1.8018578091624573 Accuracy:0.4140625\n",
      "Loss:1.9181043302571348 Accuracy:0.3671875\n",
      "Loss:1.9434155331140617 Accuracy:0.421875\n",
      "Loss:1.963539811289905 Accuracy:0.3515625\n",
      "Loss:1.7332862449427906 Accuracy:0.3984375\n",
      "Loss:2.246225598677181 Accuracy:0.3046875\n",
      "Loss:1.8481753759883623 Accuracy:0.375\n",
      "Loss:1.765560452568232 Accuracy:0.3671875\n",
      "Loss:1.8360139320812727 Accuracy:0.359375\n",
      "Loss:1.7370981878553984 Accuracy:0.4453125\n",
      "Loss:1.7841493438027225 Accuracy:0.3359375\n",
      "Loss:1.9152175587996139 Accuracy:0.3984375\n",
      "Loss:2.016884443287257 Accuracy:0.296875\n",
      "Loss:1.748235340191489 Accuracy:0.3359375\n",
      "Loss:2.1857493811634665 Accuracy:0.2578125\n",
      "Loss:2.2642912844824137 Accuracy:0.2890625\n",
      "Loss:1.7470197059456358 Accuracy:0.3515625\n",
      "Loss:1.676361743057956 Accuracy:0.3828125\n",
      "Loss:1.8490517866036225 Accuracy:0.3046875\n",
      "Loss:2.1413488015856394 Accuracy:0.328125\n",
      "Loss:1.9379988777451909 Accuracy:0.28125\n",
      "Loss:2.1254029412641144 Accuracy:0.28125\n",
      "Loss:1.7844044870108442 Accuracy:0.40625\n",
      "Loss:1.7081186408388747 Accuracy:0.3984375\n",
      "Loss:2.0223514664961852 Accuracy:0.328125\n",
      "Loss:1.9506794236107026 Accuracy:0.3828125\n",
      "Loss:1.9811498797266884 Accuracy:0.296875\n",
      "Loss:2.0706040860002215 Accuracy:0.296875\n",
      "Loss:1.7267589341771465 Accuracy:0.390625\n",
      "Loss:2.004310065827701 Accuracy:0.3671875\n",
      "Loss:1.851798106252479 Accuracy:0.34375\n",
      "Loss:2.1037099790117955 Accuracy:0.25\n",
      "Loss:1.929672282338157 Accuracy:0.296875\n",
      "Loss:1.9407434940304074 Accuracy:0.328125\n",
      "Loss:2.0248568767232586 Accuracy:0.359375\n",
      "Loss:2.2076831437792928 Accuracy:0.3359375\n",
      "Loss:1.886309536973729 Accuracy:0.3671875\n",
      "Loss:2.1817340903627374 Accuracy:0.2734375\n",
      "Loss:1.8854823982343507 Accuracy:0.3359375\n",
      "Loss:1.9740526231398208 Accuracy:0.34375\n",
      "Loss:2.101373053212157 Accuracy:0.3515625\n",
      "Loss:1.9637783309203862 Accuracy:0.328125\n",
      "Loss:2.3343705841604843 Accuracy:0.28125\n",
      "Loss:1.9757702387320584 Accuracy:0.34375\n",
      "Loss:2.0568264648290877 Accuracy:0.3359375\n",
      "Loss:1.9209276036578808 Accuracy:0.328125\n",
      "Loss:1.884104013642271 Accuracy:0.265625\n",
      "Loss:1.8725149920148683 Accuracy:0.34375\n",
      "Loss:1.7616802339643676 Accuracy:0.390625\n",
      "Loss:2.0973055020029063 Accuracy:0.390625\n",
      "Loss:1.7597718097255752 Accuracy:0.3984375\n",
      "Loss:2.0451422369800683 Accuracy:0.328125\n",
      "Loss:1.7488539130436846 Accuracy:0.3671875\n",
      "Loss:1.6021027750967005 Accuracy:0.46875\n",
      "Loss:1.770120118556934 Accuracy:0.359375\n",
      "Loss:2.0979107205809346 Accuracy:0.390625\n",
      "Loss:1.8766254892181982 Accuracy:0.375\n",
      "Loss:1.6827969009142898 Accuracy:0.3984375\n",
      "Loss:1.8741159029717203 Accuracy:0.3125\n",
      "Loss:2.0482833515419703 Accuracy:0.328125\n",
      "Loss:1.6389181401174717 Accuracy:0.375\n",
      "Loss:1.8630982955119353 Accuracy:0.3671875\n",
      "Loss:1.891603584919673 Accuracy:0.34375\n",
      "Loss:2.035834892418764 Accuracy:0.328125\n",
      "Loss:2.293780226754701 Accuracy:0.3046875\n",
      "Loss:1.809738428485146 Accuracy:0.359375\n",
      "Loss:1.7026821276217572 Accuracy:0.421875\n",
      "Loss:1.8806009136201909 Accuracy:0.390625\n",
      "Loss:2.3525933614808903 Accuracy:0.265625\n",
      "Loss:2.1071007823620524 Accuracy:0.3515625\n",
      "Loss:1.8175072468143856 Accuracy:0.3203125\n",
      "Loss:2.0633448104927554 Accuracy:0.3515625\n",
      "Loss:1.8623005752558255 Accuracy:0.3359375\n",
      "Loss:1.8139162030823839 Accuracy:0.3671875\n",
      "Loss:1.7365680594316117 Accuracy:0.328125\n",
      "Loss:1.835601866716341 Accuracy:0.40625\n",
      "Loss:1.8625360526249701 Accuracy:0.359375\n",
      "Loss:1.8877531807175876 Accuracy:0.375\n",
      "Loss:1.9952798551105422 Accuracy:0.3359375\n",
      "Loss:1.7795194623873287 Accuracy:0.3828125\n",
      "Loss:1.9202507948633567 Accuracy:0.3359375\n",
      "Loss:1.7476059077079071 Accuracy:0.375\n",
      "Loss:1.6646367183691588 Accuracy:0.3515625\n",
      "Loss:1.6374398332554567 Accuracy:0.4453125\n",
      "Loss:1.8456003071707556 Accuracy:0.390625\n",
      "Loss:1.5448506211617827 Accuracy:0.5\n",
      "Loss:1.7740353302644642 Accuracy:0.390625\n",
      "Loss:2.057360250713218 Accuracy:0.34375\n",
      "Loss:1.939938100872423 Accuracy:0.359375\n",
      "Loss:1.8710678098912263 Accuracy:0.359375\n",
      "Loss:1.7057055973822264 Accuracy:0.3671875\n",
      "Loss:1.617427039337085 Accuracy:0.453125\n",
      "Loss:1.896796555242941 Accuracy:0.390625\n",
      "Loss:1.7860838631524376 Accuracy:0.359375\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss:2.0985658984487507 Accuracy:0.390625\n",
      "Loss:2.0206780159168254 Accuracy:0.359375\n",
      "Loss:1.9185046525845082 Accuracy:0.421875\n",
      "Loss:2.1110781196975092 Accuracy:0.328125\n",
      "Loss:1.9387091014448787 Accuracy:0.40625\n",
      "Loss:1.8939051809082434 Accuracy:0.40625\n",
      "Loss:1.7974242298868202 Accuracy:0.3984375\n",
      "Loss:1.8229189354262174 Accuracy:0.3515625\n",
      "Loss:1.7786895094194413 Accuracy:0.4296875\n",
      "Loss:1.7744701431441319 Accuracy:0.390625\n",
      "Loss:1.8671399622239035 Accuracy:0.34375\n",
      "Loss:1.747750752470582 Accuracy:0.4375\n",
      "Loss:1.7342892117280475 Accuracy:0.421875\n",
      "Loss:1.8910136842575773 Accuracy:0.328125\n",
      "Loss:1.935369278598469 Accuracy:0.3671875\n",
      "Loss:1.761637301853963 Accuracy:0.4453125\n",
      "Loss:1.712927838763306 Accuracy:0.3828125\n",
      "Loss:1.959477802444463 Accuracy:0.34375\n",
      "Loss:2.0789668884236097 Accuracy:0.34375\n",
      "Loss:1.913616239104801 Accuracy:0.359375\n",
      "Loss:1.9000362456212025 Accuracy:0.3828125\n",
      "Loss:1.889815989133937 Accuracy:0.3515625\n",
      "Loss:2.065375039878809 Accuracy:0.3359375\n",
      "Loss:1.8011545934575075 Accuracy:0.359375\n",
      "Loss:1.8664659327075985 Accuracy:0.40625\n",
      "Loss:2.029508960343862 Accuracy:0.3203125\n",
      "Loss:1.8914804987667728 Accuracy:0.359375\n",
      "Loss:1.7368590842045242 Accuracy:0.4375\n",
      "Loss:1.866922315639702 Accuracy:0.3359375\n",
      "Loss:2.032641959396449 Accuracy:0.3515625\n",
      "Loss:1.8716411505784654 Accuracy:0.3359375\n",
      "Loss:1.8594101606239093 Accuracy:0.3515625\n",
      "Loss:1.8446047386443198 Accuracy:0.3671875\n",
      "Loss:1.7492022150786735 Accuracy:0.3984375\n",
      "Loss:1.6989934900726849 Accuracy:0.375\n",
      "Loss:1.9205807460363995 Accuracy:0.328125\n",
      "Loss:2.0061170680575935 Accuracy:0.3359375\n",
      "Loss:1.73033734613198 Accuracy:0.375\n",
      "Loss:1.6959991557085967 Accuracy:0.421875\n",
      "Loss:2.3679823910901474 Accuracy:0.203125\n",
      "Loss:1.83301677790809 Accuracy:0.390625\n",
      "Loss:1.74276663412591 Accuracy:0.3984375\n",
      "Loss:1.8119801507671038 Accuracy:0.40625\n",
      "Loss:1.898614300313564 Accuracy:0.296875\n",
      "Loss:1.8853061413148853 Accuracy:0.375\n",
      "Loss:1.676593195351931 Accuracy:0.4453125\n",
      "Loss:1.780398143597743 Accuracy:0.3984375\n",
      "Loss:1.8125070902155178 Accuracy:0.3359375\n",
      "Loss:1.803791223076356 Accuracy:0.390625\n",
      "Loss:1.6899437917938371 Accuracy:0.34375\n",
      "Loss:1.6674542356162219 Accuracy:0.453125\n",
      "Loss:1.7869162245106334 Accuracy:0.40625\n",
      "Loss:1.6899352122109412 Accuracy:0.40625\n",
      "Loss:1.754772808308573 Accuracy:0.40625\n",
      "Loss:1.6384434294688768 Accuracy:0.40625\n",
      "Loss:1.7672098548187307 Accuracy:0.3984375\n",
      "Loss:1.893055558824885 Accuracy:0.3671875\n",
      "Loss:1.807314252230779 Accuracy:0.3671875\n",
      "Loss:1.7631401330034557 Accuracy:0.4140625\n",
      "Loss:1.6645128203466857 Accuracy:0.40625\n",
      "Loss:1.820202978623984 Accuracy:0.3515625\n",
      "Loss:1.7505884422102889 Accuracy:0.3984375\n",
      "Loss:1.7127766824490978 Accuracy:0.4140625\n",
      "Loss:1.8845665043883795 Accuracy:0.3203125\n",
      "Loss:1.8810120384067375 Accuracy:0.3359375\n",
      "Loss:1.631208910511593 Accuracy:0.4140625\n",
      "Loss:1.939533129751841 Accuracy:0.2734375\n",
      "Loss:1.7614753941718813 Accuracy:0.3828125\n",
      "Loss:1.7675435840964866 Accuracy:0.375\n",
      "Loss:1.842945040241625 Accuracy:0.390625\n",
      "Loss:1.7069575469393055 Accuracy:0.4140625\n",
      "Loss:1.8986300162137972 Accuracy:0.3828125\n",
      "Loss:1.7216949616922288 Accuracy:0.4296875\n",
      "Loss:1.9275504003618877 Accuracy:0.390625\n",
      "Loss:1.6777723938030213 Accuracy:0.4296875\n",
      "Loss:1.945938113024404 Accuracy:0.375\n",
      "Loss:1.9489994750416944 Accuracy:0.3046875\n",
      "Loss:1.788420978572129 Accuracy:0.34375\n",
      "Loss:1.7860489250872222 Accuracy:0.40625\n",
      "Loss:1.8556065722747739 Accuracy:0.3359375\n",
      "Loss:1.5780358139292092 Accuracy:0.484375\n",
      "Loss:1.810057510859981 Accuracy:0.390625\n",
      "Loss:1.865197147839086 Accuracy:0.3984375\n",
      "Loss:2.0498828564627187 Accuracy:0.375\n",
      "Loss:1.5798510074909209 Accuracy:0.4140625\n",
      "Loss:1.8291336040270934 Accuracy:0.390625\n",
      "Loss:1.7695683762070709 Accuracy:0.3125\n",
      "Loss:1.8480786374024174 Accuracy:0.3515625\n",
      "Loss:1.7844725297718318 Accuracy:0.34375\n",
      "Loss:1.8747984661904287 Accuracy:0.296875\n",
      "Loss:1.5138962186831828 Accuracy:0.4375\n",
      "Loss:1.876236506914576 Accuracy:0.375\n",
      "Loss:1.7362001430963212 Accuracy:0.3515625\n",
      "Loss:1.832496708137556 Accuracy:0.3515625\n",
      "Loss:1.8205876143783624 Accuracy:0.34375\n",
      "Loss:1.8223778482807524 Accuracy:0.375\n",
      "Loss:1.978975334146525 Accuracy:0.3046875\n",
      "Loss:1.5426057966943563 Accuracy:0.4921875\n",
      "Loss:1.8635659980506523 Accuracy:0.359375\n",
      "Loss:1.7584660331898387 Accuracy:0.3515625\n",
      "Loss:1.8760441200120512 Accuracy:0.328125\n",
      "Loss:1.7864569975754403 Accuracy:0.3515625\n",
      "Loss:1.969306226648207 Accuracy:0.3125\n",
      "Loss:1.6407683404581976 Accuracy:0.4375\n",
      "Loss:1.7467986817923955 Accuracy:0.3828125\n",
      "Loss:1.843703062875731 Accuracy:0.3671875\n",
      "Loss:1.8168347440914854 Accuracy:0.40625\n",
      "Loss:1.598054995816883 Accuracy:0.4609375\n",
      "Loss:2.15497859934138 Accuracy:0.2734375\n",
      "Loss:1.4888397832583125 Accuracy:0.4921875\n",
      "Loss:1.6674295098090877 Accuracy:0.34375\n",
      "Loss:1.7557813470450145 Accuracy:0.46875\n",
      "Loss:2.1115035316963233 Accuracy:0.3046875\n",
      "Loss:1.8642260633296293 Accuracy:0.3515625\n",
      "Loss:1.7516016010676987 Accuracy:0.4140625\n",
      "Loss:1.7697526397115562 Accuracy:0.34375\n",
      "Loss:1.7153520102434152 Accuracy:0.3828125\n",
      "Loss:1.6323988489041732 Accuracy:0.40625\n",
      "Loss:1.8568098793769667 Accuracy:0.4140625\n",
      "Loss:1.5444944788961652 Accuracy:0.4453125\n",
      "Loss:1.6479210064693146 Accuracy:0.4140625\n",
      "Loss:1.918353125837405 Accuracy:0.3984375\n",
      "Loss:2.1394438682484327 Accuracy:0.3203125\n",
      "Loss:1.8562483154142386 Accuracy:0.3671875\n",
      "Loss:1.6029905670383235 Accuracy:0.4609375\n",
      "Loss:2.0063415590594227 Accuracy:0.3828125\n",
      "Loss:1.9961958564577502 Accuracy:0.3515625\n",
      "Loss:1.8665169647226008 Accuracy:0.359375\n",
      "Loss:1.7072562150863577 Accuracy:0.40625\n",
      "Loss:1.860690693873751 Accuracy:0.3359375\n",
      "Loss:2.224075515827919 Accuracy:0.265625\n",
      "Loss:2.0097815398735706 Accuracy:0.390625\n",
      "Loss:1.79999931355164 Accuracy:0.34375\n",
      "Loss:1.9204984563211975 Accuracy:0.359375\n",
      "Loss:1.9425446194566867 Accuracy:0.265625\n",
      "Loss:1.9463086079182559 Accuracy:0.359375\n",
      "Loss:1.9379218948793706 Accuracy:0.3359375\n",
      "Loss:1.8122753346659377 Accuracy:0.3515625\n",
      "Loss:1.9626113044625917 Accuracy:0.34375\n",
      "Loss:1.7813136463403823 Accuracy:0.40625\n",
      "Loss:2.126821630078265 Accuracy:0.28125\n",
      "Loss:1.9148964964066584 Accuracy:0.3125\n",
      "Loss:1.8154624980405956 Accuracy:0.3984375\n",
      "Loss:1.9810779405221894 Accuracy:0.328125\n",
      "Loss:1.9405170493024797 Accuracy:0.3984375\n",
      "Loss:1.7685745759710674 Accuracy:0.4140625\n",
      "Loss:1.9017794922329196 Accuracy:0.390625\n",
      "Loss:1.8218931258733504 Accuracy:0.390625\n",
      "Loss:1.7968347558315998 Accuracy:0.3046875\n",
      "Loss:1.671269028959698 Accuracy:0.359375\n",
      "Loss:1.970605162551438 Accuracy:0.328125\n",
      "Loss:1.9322288647925931 Accuracy:0.3359375\n",
      "Loss:1.8499749310392977 Accuracy:0.3671875\n",
      "Loss:1.5777261334178045 Accuracy:0.4375\n",
      "Loss:1.6717638096744056 Accuracy:0.3359375\n",
      "Loss:2.0270476611879604 Accuracy:0.359375\n",
      "Loss:1.594991326392851 Accuracy:0.390625\n",
      "Loss:1.695654351118054 Accuracy:0.4296875\n",
      "Loss:1.670540995180579 Accuracy:0.4375\n",
      "Loss:1.7639470243931967 Accuracy:0.4453125\n",
      "Loss:1.7107151453344291 Accuracy:0.4296875\n",
      "Loss:1.6862285353677708 Accuracy:0.421875\n",
      "Loss:1.7211930104107893 Accuracy:0.3515625\n",
      "Loss:1.7680792420191054 Accuracy:0.3515625\n",
      "Loss:1.7217160034269159 Accuracy:0.3515625\n",
      "Loss:1.6377236797536843 Accuracy:0.4140625\n",
      "Loss:1.8487219520238045 Accuracy:0.359375\n",
      "Loss:1.7245480153216746 Accuracy:0.4609375\n",
      "Loss:1.632870562262569 Accuracy:0.3984375\n",
      "Loss:1.9076960175640758 Accuracy:0.3515625\n",
      "Loss:1.8347589589198554 Accuracy:0.3828125\n",
      "Loss:1.6404439514814078 Accuracy:0.390625\n",
      "Loss:1.6810562223793122 Accuracy:0.359375\n",
      "Loss:1.7618371984749603 Accuracy:0.4140625\n",
      "Loss:1.8156347513984237 Accuracy:0.375\n",
      "Loss:1.7846250396163472 Accuracy:0.3515625\n",
      "Loss:1.7327256520690755 Accuracy:0.3515625\n",
      "Loss:1.7702109436881952 Accuracy:0.3828125\n",
      "Loss:1.7052210121429225 Accuracy:0.3671875\n",
      "Loss:1.8826301574845714 Accuracy:0.34375\n",
      "Loss:1.7807134775389644 Accuracy:0.4140625\n",
      "Loss:1.6609203531192422 Accuracy:0.40625\n",
      "Loss:1.675360033863276 Accuracy:0.40625\n",
      "Loss:1.7516786129122384 Accuracy:0.3671875\n",
      "Loss:1.6140606288093629 Accuracy:0.4609375\n",
      "Loss:1.557620752502951 Accuracy:0.4453125\n",
      "Loss:1.6130963084065484 Accuracy:0.4453125\n",
      "Loss:1.631779400659938 Accuracy:0.4140625\n",
      "Loss:2.011008890006272 Accuracy:0.34375\n",
      "Loss:1.7043302830790348 Accuracy:0.3671875\n",
      "Loss:1.691992747262463 Accuracy:0.4296875\n",
      "Loss:1.6383857690188242 Accuracy:0.390625\n",
      "Loss:1.5462856964724487 Accuracy:0.4296875\n",
      "Loss:1.7253954428772886 Accuracy:0.3359375\n",
      "Loss:1.5590412544653895 Accuracy:0.40625\n",
      "Loss:1.6046590908619545 Accuracy:0.4375\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss:1.6759374756884506 Accuracy:0.4140625\n",
      "Loss:1.611140568992953 Accuracy:0.4453125\n",
      "Loss:1.6994032047692784 Accuracy:0.3828125\n",
      "Loss:1.9314975477638185 Accuracy:0.328125\n",
      "Loss:1.7209410747825191 Accuracy:0.4296875\n",
      "Loss:1.858489669800643 Accuracy:0.359375\n",
      "Loss:1.8365011990171611 Accuracy:0.359375\n",
      "Loss:1.6780417440907898 Accuracy:0.4375\n",
      "Loss:1.7006700594703128 Accuracy:0.4296875\n",
      "Loss:1.705424186252218 Accuracy:0.390625\n",
      "Loss:1.6972347800542589 Accuracy:0.4375\n",
      "Loss:1.7416040443762641 Accuracy:0.4296875\n",
      "Loss:1.736091555045045 Accuracy:0.4375\n",
      "Loss:1.8123941361309366 Accuracy:0.390625\n",
      "Loss:1.6494202195898686 Accuracy:0.40625\n",
      "Loss:1.494399677820817 Accuracy:0.4453125\n",
      "Loss:1.671141531751593 Accuracy:0.421875\n",
      "Loss:2.048142699336039 Accuracy:0.28125\n",
      "Loss:1.5591904582917944 Accuracy:0.4296875\n",
      "Loss:1.8220864784915454 Accuracy:0.3671875\n",
      "Loss:1.703716983140434 Accuracy:0.453125\n",
      "Loss:1.7327803232004433 Accuracy:0.390625\n",
      "Loss:1.7456391086320717 Accuracy:0.3984375\n",
      "Loss:1.709526278989144 Accuracy:0.375\n",
      "Loss:1.4881672958804348 Accuracy:0.515625\n",
      "Loss:1.4110375701259243 Accuracy:0.4765625\n",
      "Loss:1.6046798319001796 Accuracy:0.4453125\n",
      "Loss:1.8565725498231938 Accuracy:0.3515625\n",
      "Loss:1.707156223424205 Accuracy:0.3671875\n",
      "Loss:1.8599036725551632 Accuracy:0.390625\n",
      "Loss:1.5084519651072854 Accuracy:0.484375\n",
      "Loss:1.69094407119609 Accuracy:0.4453125\n",
      "Loss:1.558115584613014 Accuracy:0.4765625\n",
      "Loss:1.8224405768194865 Accuracy:0.375\n",
      "Loss:1.6510973359008227 Accuracy:0.375\n",
      "Loss:1.7510250907301814 Accuracy:0.4140625\n",
      "Loss:1.840432398979481 Accuracy:0.3984375\n",
      "Loss:1.7683181064173752 Accuracy:0.3671875\n",
      "Loss:1.6017791321028587 Accuracy:0.453125\n",
      "Loss:1.9265986221118436 Accuracy:0.375\n",
      "Loss:1.8678359497975485 Accuracy:0.359375\n",
      "Loss:1.7020666755551521 Accuracy:0.359375\n",
      "Loss:1.7315281240846474 Accuracy:0.4296875\n",
      "Loss:1.716650091683837 Accuracy:0.390625\n",
      "Loss:1.622131051595502 Accuracy:0.4296875\n",
      "Loss:1.9256102066059837 Accuracy:0.3515625\n",
      "Loss:1.8020056891523244 Accuracy:0.3203125\n",
      "Loss:1.5164467431892215 Accuracy:0.4296875\n",
      "Loss:1.750591635753507 Accuracy:0.3671875\n",
      "Loss:2.0528017833271863 Accuracy:0.3125\n",
      "Loss:1.6811028895934639 Accuracy:0.4609375\n",
      "Loss:1.8595262871545841 Accuracy:0.3828125\n",
      "Loss:1.7840778180617685 Accuracy:0.3828125\n",
      "Loss:1.7485557350732708 Accuracy:0.421875\n",
      "Loss:1.7121788463801433 Accuracy:0.3671875\n",
      "Loss:1.7152179152087497 Accuracy:0.4140625\n",
      "Loss:1.695915834128484 Accuracy:0.421875\n",
      "Loss:1.6724028021071293 Accuracy:0.40625\n",
      "Loss:1.5516736439214118 Accuracy:0.4453125\n",
      "Loss:1.6919742777934341 Accuracy:0.375\n",
      "Loss:1.7296341647139317 Accuracy:0.3671875\n",
      "Loss:1.7699049378740561 Accuracy:0.4140625\n",
      "Loss:1.7469330853329383 Accuracy:0.4375\n",
      "Loss:1.7482658531974828 Accuracy:0.328125\n",
      "Loss:1.597645722271637 Accuracy:0.421875\n",
      "Loss:1.896885340766255 Accuracy:0.390625\n",
      "Loss:1.7437343847945237 Accuracy:0.40625\n",
      "Loss:1.7303985795075403 Accuracy:0.4296875\n",
      "Loss:1.6666854905798933 Accuracy:0.375\n",
      "Loss:1.6648540680838517 Accuracy:0.4609375\n",
      "Loss:1.9368675211549489 Accuracy:0.375\n",
      "Loss:1.628214194252049 Accuracy:0.40625\n",
      "Loss:1.7853065661986292 Accuracy:0.3828125\n",
      "Loss:1.70534345562977 Accuracy:0.453125\n",
      "Loss:1.6927039583990853 Accuracy:0.328125\n",
      "Loss:1.6399358917184852 Accuracy:0.3984375\n",
      "Loss:1.4278218503327635 Accuracy:0.5234375\n",
      "Loss:1.9816211105365842 Accuracy:0.3359375\n",
      "Loss:1.622095953718848 Accuracy:0.421875\n",
      "Loss:1.631693111066548 Accuracy:0.46875\n",
      "Loss:1.6011116116140243 Accuracy:0.4921875\n",
      "Loss:1.827948019058872 Accuracy:0.3359375\n",
      "Loss:1.5099393493728503 Accuracy:0.4453125\n",
      "Loss:1.6142309836449766 Accuracy:0.40625\n",
      "Loss:1.6748168223317577 Accuracy:0.40625\n",
      "Loss:1.5801845414935438 Accuracy:0.453125\n",
      "Loss:1.7191973350470962 Accuracy:0.375\n",
      "Loss:1.6897886667152449 Accuracy:0.375\n",
      "Loss:1.8877494851411587 Accuracy:0.328125\n",
      "Loss:1.5899743145650196 Accuracy:0.4609375\n",
      "Loss:1.8097871173457496 Accuracy:0.40625\n",
      "Loss:1.6526882397888365 Accuracy:0.4296875\n",
      "Loss:1.6639485712980506 Accuracy:0.3671875\n",
      "Loss:1.7035390076100496 Accuracy:0.40625\n",
      "Loss:1.7896691045079647 Accuracy:0.375\n",
      "Loss:1.8756261662050366 Accuracy:0.421875\n",
      "Loss:1.7535566824017592 Accuracy:0.375\n",
      "Loss:1.8147990702492667 Accuracy:0.328125\n",
      "Loss:1.781353696730403 Accuracy:0.3828125\n",
      "Loss:1.6108648464949002 Accuracy:0.3984375\n",
      "Loss:1.9089468448862856 Accuracy:0.3359375\n",
      "Loss:1.423587250828369 Accuracy:0.484375\n",
      "Loss:1.62538982263646 Accuracy:0.421875\n",
      "Loss:1.6423571796590188 Accuracy:0.421875\n",
      "Loss:1.705165883341674 Accuracy:0.4375\n",
      "Loss:1.4958345715144161 Accuracy:0.4453125\n",
      "Loss:1.9191153830008347 Accuracy:0.28125\n",
      "Loss:1.6281769280291303 Accuracy:0.4296875\n",
      "Loss:1.7194495860566228 Accuracy:0.3671875\n",
      "Loss:1.6350113439184715 Accuracy:0.4375\n",
      "Loss:1.7311877684463228 Accuracy:0.375\n",
      "Loss:1.631166792765726 Accuracy:0.3984375\n",
      "Loss:1.4598335141892655 Accuracy:0.484375\n",
      "Loss:1.54798625615666 Accuracy:0.515625\n",
      "Loss:1.855057259696777 Accuracy:0.3671875\n",
      "Loss:1.6474310104704688 Accuracy:0.40625\n",
      "Loss:1.6440562793897477 Accuracy:0.4296875\n",
      "Loss:1.5815668524800657 Accuracy:0.4765625\n",
      "Loss:1.602790142985883 Accuracy:0.4296875\n",
      "Loss:1.6805686190374112 Accuracy:0.40625\n",
      "Loss:1.6876247168729606 Accuracy:0.4140625\n",
      "Loss:1.599693243668348 Accuracy:0.4609375\n",
      "Loss:1.5104688624275666 Accuracy:0.3984375\n",
      "Loss:1.3788555017363975 Accuracy:0.4921875\n",
      "Loss:1.7112952771209031 Accuracy:0.4453125\n",
      "Loss:1.7795730320268583 Accuracy:0.3828125\n",
      "Loss:1.6691926152975163 Accuracy:0.359375\n",
      "Loss:1.6977541848290065 Accuracy:0.4296875\n",
      "Loss:1.7198414672422828 Accuracy:0.359375\n",
      "Loss:1.6738907321619942 Accuracy:0.4296875\n",
      "Loss:1.6186281389580928 Accuracy:0.5\n",
      "Loss:1.7557778160865265 Accuracy:0.4453125\n",
      "Loss:1.8294633139949141 Accuracy:0.4296875\n",
      "Loss:1.812583806467771 Accuracy:0.3828125\n",
      "Loss:1.7707770090492234 Accuracy:0.3828125\n",
      "Loss:1.7729807653614587 Accuracy:0.40625\n",
      "Loss:1.7085735961744726 Accuracy:0.390625\n",
      "Loss:1.5775765456968054 Accuracy:0.453125\n",
      "Loss:1.6900062323716534 Accuracy:0.3984375\n",
      "Loss:1.8713248052435558 Accuracy:0.3828125\n",
      "Loss:1.7740523314432894 Accuracy:0.3984375\n",
      "Loss:1.8082337795500572 Accuracy:0.3671875\n",
      "Loss:1.9045939665169258 Accuracy:0.3125\n",
      "Loss:1.869177816141664 Accuracy:0.3125\n",
      "Loss:1.590578803966126 Accuracy:0.3984375\n",
      "Loss:1.5740152512962031 Accuracy:0.4375\n",
      "Loss:1.7419449841965378 Accuracy:0.359375\n",
      "Loss:1.504218743149396 Accuracy:0.484375\n",
      "Loss:1.8007806579813324 Accuracy:0.375\n",
      "Loss:1.9409242931765849 Accuracy:0.3984375\n",
      "Loss:1.8148898702214125 Accuracy:0.3671875\n",
      "Loss:1.7902381546309267 Accuracy:0.375\n",
      "Loss:1.3975319737045782 Accuracy:0.515625\n",
      "Loss:1.8170958972634659 Accuracy:0.359375\n",
      "Loss:1.8082572236011978 Accuracy:0.3515625\n",
      "Loss:1.5994842026927623 Accuracy:0.4453125\n",
      "Loss:1.6032666877921975 Accuracy:0.4375\n",
      "Loss:2.0077654104951654 Accuracy:0.3203125\n",
      "Loss:1.5130860560109833 Accuracy:0.4453125\n",
      "Loss:1.5645572985173728 Accuracy:0.484375\n",
      "Loss:1.6856879946045946 Accuracy:0.3828125\n",
      "Loss:2.005994352157132 Accuracy:0.328125\n",
      "Loss:1.6630995061766205 Accuracy:0.4375\n",
      "Loss:1.505841441365237 Accuracy:0.4296875\n",
      "Loss:1.9072271376900105 Accuracy:0.296875\n",
      "Loss:1.6584493293094869 Accuracy:0.484375\n",
      "Loss:1.482590672274286 Accuracy:0.40625\n",
      "Loss:1.6629826813512711 Accuracy:0.3359375\n",
      "Loss:1.7951355503665405 Accuracy:0.4296875\n",
      "Loss:1.5926004792499326 Accuracy:0.4609375\n",
      "Loss:1.6880657648856303 Accuracy:0.3984375\n",
      "Loss:1.9334333781238247 Accuracy:0.3203125\n",
      "Loss:1.7132962097810318 Accuracy:0.4375\n",
      "Loss:1.4850755903132693 Accuracy:0.4375\n",
      "Loss:1.6238327066257954 Accuracy:0.453125\n",
      "Loss:1.7513679662612536 Accuracy:0.359375\n",
      "Loss:1.710437335960565 Accuracy:0.4140625\n",
      "Loss:1.6920621809611047 Accuracy:0.40625\n",
      "Loss:1.627101491981565 Accuracy:0.4375\n",
      "Loss:1.5163886181305457 Accuracy:0.4609375\n",
      "Loss:1.66371563519398 Accuracy:0.4140625\n",
      "Loss:1.7400346182066009 Accuracy:0.4453125\n",
      "Loss:1.8647217418599669 Accuracy:0.3828125\n",
      "Loss:1.5900668408674896 Accuracy:0.3984375\n",
      "Loss:1.6103270516135435 Accuracy:0.4453125\n",
      "Loss:1.817094186835401 Accuracy:0.34375\n",
      "Loss:1.6536603591318826 Accuracy:0.3828125\n",
      "Loss:1.7468991079982703 Accuracy:0.3671875\n",
      "Loss:1.468236327323434 Accuracy:0.46875\n",
      "Loss:1.5831820646983576 Accuracy:0.453125\n",
      "Loss:2.0437614410926512 Accuracy:0.375\n",
      "Loss:1.5689022605075391 Accuracy:0.484375\n",
      "Loss:1.5203898726620433 Accuracy:0.4453125\n",
      "Loss:1.6895426187984868 Accuracy:0.421875\n",
      "Loss:1.5720098599534047 Accuracy:0.4296875\n",
      "Loss:1.5417591282151761 Accuracy:0.4609375\n",
      "Loss:1.5732196630663666 Accuracy:0.4453125\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss:1.6770121104284708 Accuracy:0.4453125\n",
      "Loss:1.5034586320228691 Accuracy:0.46875\n",
      "Loss:1.7778427956625578 Accuracy:0.3515625\n",
      "Loss:1.7252432614998807 Accuracy:0.375\n",
      "Loss:1.8063843842476732 Accuracy:0.375\n",
      "Loss:1.585090050034842 Accuracy:0.4453125\n",
      "Loss:1.494383034526737 Accuracy:0.5390625\n",
      "Loss:1.7511706479246474 Accuracy:0.3359375\n",
      "Loss:1.5071417103838343 Accuracy:0.5\n",
      "Loss:1.6318107676270777 Accuracy:0.3828125\n",
      "Loss:1.7545597593237323 Accuracy:0.359375\n",
      "Loss:1.752347432283844 Accuracy:0.4140625\n",
      "Loss:1.848203594773958 Accuracy:0.375\n",
      "Loss:1.4865852709551481 Accuracy:0.4375\n",
      "Loss:1.5816745913010704 Accuracy:0.4140625\n",
      "Loss:1.7779420420372027 Accuracy:0.4375\n",
      "Loss:1.5638936488552102 Accuracy:0.4921875\n",
      "Loss:1.608345048845829 Accuracy:0.4375\n",
      "Loss:1.627075101880483 Accuracy:0.4375\n",
      "Loss:1.7054506356054164 Accuracy:0.40625\n",
      "Loss:1.7600207271045938 Accuracy:0.4140625\n",
      "Loss:1.6522458454238946 Accuracy:0.375\n",
      "Loss:1.606211360649283 Accuracy:0.4140625\n",
      "Loss:1.7982358191601313 Accuracy:0.34375\n",
      "Loss:1.3964586074670162 Accuracy:0.453125\n",
      "Loss:1.689707619102046 Accuracy:0.4140625\n",
      "Loss:1.7767411417577468 Accuracy:0.3984375\n",
      "Loss:1.592710865337778 Accuracy:0.421875\n",
      "Loss:1.7455896923777126 Accuracy:0.3984375\n",
      "Loss:1.5304124251621003 Accuracy:0.4453125\n",
      "Loss:1.8601992671989858 Accuracy:0.4140625\n",
      "Loss:1.7802137721106743 Accuracy:0.3984375\n",
      "Loss:1.7534596783112506 Accuracy:0.3671875\n",
      "Loss:1.4792376758437387 Accuracy:0.4921875\n",
      "Loss:1.7209556083136364 Accuracy:0.359375\n",
      "Loss:1.6745270932096599 Accuracy:0.3984375\n",
      "Loss:1.9080018411114792 Accuracy:0.3671875\n",
      "Loss:1.874798294216367 Accuracy:0.3515625\n",
      "Loss:1.7660364650528035 Accuracy:0.359375\n",
      "Loss:1.696042779040444 Accuracy:0.46875\n",
      "Loss:1.6952723567409387 Accuracy:0.46875\n",
      "Loss:1.6666462071491521 Accuracy:0.3984375\n",
      "Loss:1.5336790571997043 Accuracy:0.4453125\n",
      "Loss:2.0839864849157714 Accuracy:0.3046875\n",
      "Loss:1.547697513693625 Accuracy:0.40625\n",
      "Loss:1.6922623768327532 Accuracy:0.390625\n",
      "Loss:1.8699826232701455 Accuracy:0.3984375\n",
      "Loss:1.4277414512333702 Accuracy:0.4296875\n",
      "Loss:1.705486126009769 Accuracy:0.4140625\n",
      "Loss:1.5879402575353856 Accuracy:0.4609375\n",
      "Loss:1.9358969431700253 Accuracy:0.40625\n",
      "Loss:1.690080034061024 Accuracy:0.3828125\n",
      "Loss:1.726024067534195 Accuracy:0.34375\n",
      "Loss:1.6284800240668922 Accuracy:0.4453125\n"
     ]
    }
   ],
   "source": [
    "# Train mini batch\n",
    "\n",
    "net = FC(in_shape=X.shape[1:], out_size=Y.shape[1])\n",
    "criteria = Criteria(mode='cross_entropy')\n",
    "optimizer = Adam(net,learning_rate=0.01)\n",
    "\n",
    "\n",
    "for i in range(num_iter):\n",
    "    \n",
    "    permutation = np.random.permutation(m)\n",
    "    \n",
    "    for j in range(0,m,batch_size):\n",
    "        \n",
    "        indices = permutation[j:j+batch_size]\n",
    "        X_batch, Y_batch = X[indices], Y[indices]\n",
    "        \n",
    "        A = net.forward(X_batch)\n",
    "\n",
    "        loss = criteria.forward(A, Y_batch)\n",
    "        \n",
    "        dZ = criteria.backward()\n",
    "        dZ = net.backward(dZ)\n",
    "\n",
    "        optimizer.step(net)\n",
    "        \n",
    "        accuracy = np.sum(np.argmax(A,axis=1)==np.argmax(Y_batch,axis=1))/len(X_batch)\n",
    "        print(f\"Loss:{loss} Accuracy:{accuracy}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Num test:6000 Loss:1.7603395872698047 Accuracy:0.395\n"
     ]
    }
   ],
   "source": [
    "A = net.forward(X_test)\n",
    "loss = criteria.forward(A, Y_test)\n",
    "accuracy = np.sum(np.argmax(A,axis=1)==np.argmax(Y_test,axis=1))/len(X_test)\n",
    "\n",
    "print(f\" Num test:{len(X_test)} Loss:{loss} Accuracy:{accuracy}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "The learning plateu around "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convolution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss:3.731197510708201 Accuracy:0.0859375\n",
      "Loss:12.506931139075334 Accuracy:0.125\n",
      "Loss:14.493052762242199 Accuracy:0.1328125\n",
      "Loss:15.917773309171132 Accuracy:0.1171875\n",
      "Loss:15.83547874207524 Accuracy:0.1015625\n",
      "Loss:15.21398374473239 Accuracy:0.140625\n",
      "Loss:15.948972219582611 Accuracy:0.1015625\n",
      "Loss:16.092949672544798 Accuracy:0.1171875\n",
      "Loss:16.122367646367003 Accuracy:0.1015625\n",
      "Loss:15.70363710340074 Accuracy:0.125\n",
      "Loss:14.124163419207635 Accuracy:0.1796875\n",
      "Loss:13.785925673294434 Accuracy:0.1875\n",
      "Loss:12.475402357633381 Accuracy:0.265625\n",
      "Loss:12.80175119407701 Accuracy:0.1796875\n",
      "Loss:12.513352553525845 Accuracy:0.15625\n",
      "Loss:11.557290179390346 Accuracy:0.203125\n",
      "Loss:15.599379831499565 Accuracy:0.1171875\n",
      "Loss:14.849347004326578 Accuracy:0.1328125\n",
      "Loss:14.188231464756296 Accuracy:0.109375\n",
      "Loss:13.076504214342597 Accuracy:0.1640625\n",
      "Loss:11.751334383836285 Accuracy:0.234375\n",
      "Loss:12.050431268652869 Accuracy:0.2109375\n",
      "Loss:11.764164681486253 Accuracy:0.2265625\n",
      "Loss:11.296836474781731 Accuracy:0.2265625\n",
      "Loss:10.85839091103873 Accuracy:0.21875\n",
      "Loss:10.787527506703391 Accuracy:0.1953125\n",
      "Loss:10.030600951004125 Accuracy:0.2890625\n",
      "Loss:11.012373991748476 Accuracy:0.203125\n",
      "Loss:9.736375565361396 Accuracy:0.2109375\n",
      "Loss:9.632110743744661 Accuracy:0.171875\n",
      "Loss:9.215478484535716 Accuracy:0.2109375\n",
      "Loss:8.33340437882399 Accuracy:0.296875\n",
      "Loss:7.796756664587723 Accuracy:0.2890625\n",
      "Loss:8.225987694474645 Accuracy:0.234375\n",
      "Loss:9.520254861963567 Accuracy:0.234375\n",
      "Loss:8.398823913322127 Accuracy:0.234375\n",
      "Loss:7.769121527483692 Accuracy:0.234375\n",
      "Loss:7.365703735176277 Accuracy:0.296875\n",
      "Loss:8.551209612400285 Accuracy:0.2109375\n",
      "Loss:7.139117367900481 Accuracy:0.2421875\n",
      "Loss:6.32846405428686 Accuracy:0.3046875\n",
      "Loss:7.392850227576012 Accuracy:0.2265625\n",
      "Loss:5.809191079840889 Accuracy:0.265625\n",
      "Loss:5.947700905917448 Accuracy:0.2734375\n",
      "Loss:5.231783363080967 Accuracy:0.2890625\n",
      "Loss:7.092655130263732 Accuracy:0.1953125\n",
      "Loss:5.274537525253377 Accuracy:0.2578125\n",
      "Loss:5.411520364811923 Accuracy:0.2421875\n",
      "Loss:5.003240336543385 Accuracy:0.375\n",
      "Loss:6.586247647957595 Accuracy:0.28125\n",
      "Loss:6.305039412358621 Accuracy:0.2578125\n",
      "Loss:4.95221438380435 Accuracy:0.234375\n",
      "Loss:5.738173267022271 Accuracy:0.234375\n",
      "Loss:4.75641763078468 Accuracy:0.28125\n",
      "Loss:5.094906805953157 Accuracy:0.2578125\n",
      "Loss:4.9997426768344475 Accuracy:0.2265625\n",
      "Loss:5.42407889456693 Accuracy:0.234375\n",
      "Loss:4.210700479479383 Accuracy:0.328125\n",
      "Loss:5.3469759776575625 Accuracy:0.1796875\n",
      "Loss:4.3298381568928015 Accuracy:0.28125\n",
      "Loss:4.2041818364389405 Accuracy:0.2734375\n",
      "Loss:4.4527960767791885 Accuracy:0.328125\n",
      "Loss:5.034614038805567 Accuracy:0.28125\n",
      "Loss:4.890269895336765 Accuracy:0.1875\n",
      "Loss:4.335167549356204 Accuracy:0.28125\n",
      "Loss:4.207921815688806 Accuracy:0.25\n",
      "Loss:4.377819969857864 Accuracy:0.265625\n",
      "Loss:4.518367161725304 Accuracy:0.2265625\n",
      "Loss:3.6348642791057837 Accuracy:0.296875\n",
      "Loss:3.912947931167868 Accuracy:0.2109375\n",
      "Loss:4.105616270812716 Accuracy:0.25\n",
      "Loss:3.3521469014408716 Accuracy:0.3046875\n",
      "Loss:4.183849818377711 Accuracy:0.1953125\n",
      "Loss:2.9022452230478075 Accuracy:0.2421875\n",
      "Loss:3.315476607415521 Accuracy:0.359375\n",
      "Loss:3.5875465486759666 Accuracy:0.234375\n",
      "Loss:3.268959477147709 Accuracy:0.2890625\n",
      "Loss:4.104344103732395 Accuracy:0.296875\n",
      "Loss:4.184561096010369 Accuracy:0.2421875\n",
      "Loss:3.470736097638903 Accuracy:0.2578125\n",
      "Loss:3.0886437966202034 Accuracy:0.25\n",
      "Loss:2.7188724315318593 Accuracy:0.3125\n",
      "Loss:2.600970519322696 Accuracy:0.390625\n",
      "Loss:3.363777145644618 Accuracy:0.25\n",
      "Loss:3.0413574535428634 Accuracy:0.2734375\n",
      "Loss:2.6518948436652385 Accuracy:0.328125\n",
      "Loss:2.4993651641766133 Accuracy:0.3828125\n",
      "Loss:3.0219654870415997 Accuracy:0.2421875\n",
      "Loss:3.049957386361029 Accuracy:0.21875\n",
      "Loss:3.153612640687932 Accuracy:0.2265625\n",
      "Loss:2.892133999610993 Accuracy:0.25\n",
      "Loss:2.96713874130276 Accuracy:0.265625\n",
      "Loss:2.726517630653709 Accuracy:0.3203125\n",
      "Loss:2.8025090284412704 Accuracy:0.3203125\n",
      "Loss:2.835196119978097 Accuracy:0.296875\n",
      "Loss:2.6816931000438933 Accuracy:0.28125\n",
      "Loss:3.010215907040793 Accuracy:0.2265625\n",
      "Loss:2.69990557667532 Accuracy:0.25\n",
      "Loss:2.560836964225552 Accuracy:0.3515625\n",
      "Loss:2.9443088304596348 Accuracy:0.265625\n",
      "Loss:2.9646867954647425 Accuracy:0.2421875\n",
      "Loss:2.7621488605963327 Accuracy:0.234375\n",
      "Loss:2.451596985351578 Accuracy:0.34375\n",
      "Loss:2.7488709345819 Accuracy:0.28125\n",
      "Loss:3.1015956677735463 Accuracy:0.21875\n",
      "Loss:2.511260795016214 Accuracy:0.2578125\n",
      "Loss:2.278663269157434 Accuracy:0.3515625\n",
      "Loss:2.6376657087215998 Accuracy:0.3203125\n",
      "Loss:2.614833065047087 Accuracy:0.2109375\n",
      "Loss:2.5265974450274644 Accuracy:0.3046875\n",
      "Loss:2.2491974325794146 Accuracy:0.2890625\n",
      "Loss:2.4150614548770606 Accuracy:0.3359375\n",
      "Loss:2.5806723662862536 Accuracy:0.2890625\n",
      "Loss:2.189857872618342 Accuracy:0.296875\n",
      "Loss:2.3428421817908966 Accuracy:0.2890625\n",
      "Loss:2.157990519964179 Accuracy:0.328125\n",
      "Loss:2.3253023429133446 Accuracy:0.296875\n",
      "Loss:2.1986749978219917 Accuracy:0.3125\n",
      "Loss:2.071328595113992 Accuracy:0.296875\n",
      "Loss:2.2047431781546 Accuracy:0.328125\n",
      "Loss:2.5605284069250396 Accuracy:0.2578125\n",
      "Loss:2.347229881055755 Accuracy:0.2890625\n",
      "Loss:2.423469229102867 Accuracy:0.296875\n",
      "Loss:2.0488639916719924 Accuracy:0.359375\n",
      "Loss:2.1090512640740235 Accuracy:0.3515625\n",
      "Loss:2.0148080205921417 Accuracy:0.34375\n",
      "Loss:2.212038016871058 Accuracy:0.3515625\n",
      "Loss:2.115655046488094 Accuracy:0.3515625\n",
      "Loss:1.9760801502609744 Accuracy:0.328125\n",
      "Loss:2.286189539879 Accuracy:0.296875\n",
      "Loss:1.9870173648540952 Accuracy:0.34375\n",
      "Loss:2.3960155373838092 Accuracy:0.28125\n",
      "Loss:2.1049555022234303 Accuracy:0.28125\n",
      "Loss:2.01437716469551 Accuracy:0.328125\n",
      "Loss:1.901231766148813 Accuracy:0.390625\n",
      "Loss:2.267800720538559 Accuracy:0.296875\n",
      "Loss:2.389912847296295 Accuracy:0.3359375\n",
      "Loss:2.2413131055019795 Accuracy:0.34375\n",
      "Loss:1.9656735412577244 Accuracy:0.390625\n",
      "Loss:2.38810125328561 Accuracy:0.2421875\n",
      "Loss:2.001686590605602 Accuracy:0.2890625\n",
      "Loss:2.1218598048540205 Accuracy:0.3046875\n",
      "Loss:2.2891956886349805 Accuracy:0.34375\n",
      "Loss:2.2540004595778615 Accuracy:0.3125\n",
      "Loss:2.049618537070177 Accuracy:0.3671875\n",
      "Loss:2.1052372579339167 Accuracy:0.3359375\n",
      "Loss:1.9129159763254724 Accuracy:0.34375\n",
      "Loss:2.215255088276444 Accuracy:0.265625\n",
      "Loss:1.8439039661434007 Accuracy:0.375\n",
      "Loss:2.3725180641367904 Accuracy:0.28125\n",
      "Loss:1.9427805109491056 Accuracy:0.3671875\n",
      "Loss:2.239535466208153 Accuracy:0.265625\n",
      "Loss:2.220119302103735 Accuracy:0.359375\n",
      "Loss:2.184357798081425 Accuracy:0.34375\n",
      "Loss:2.087480956682527 Accuracy:0.375\n",
      "Loss:2.10199828008797 Accuracy:0.3671875\n",
      "Loss:1.982775790313534 Accuracy:0.3515625\n",
      "Loss:2.069268295239399 Accuracy:0.3046875\n",
      "Loss:2.2734233238304977 Accuracy:0.28125\n",
      "Loss:2.0673155004325965 Accuracy:0.3359375\n",
      "Loss:1.8422459863436742 Accuracy:0.375\n",
      "Loss:2.195737249776542 Accuracy:0.328125\n",
      "Loss:2.3129891879445554 Accuracy:0.3359375\n",
      "Loss:1.7951188020777 Accuracy:0.421875\n",
      "Loss:2.016042008308342 Accuracy:0.3359375\n",
      "Loss:2.267931153625904 Accuracy:0.3359375\n",
      "Loss:2.341739613586624 Accuracy:0.28125\n",
      "Loss:2.130354882990003 Accuracy:0.3046875\n",
      "Loss:1.9841401886929377 Accuracy:0.390625\n",
      "Loss:1.9687765868183602 Accuracy:0.328125\n",
      "Loss:2.150902738025306 Accuracy:0.421875\n",
      "Loss:1.7610157272864162 Accuracy:0.375\n",
      "Loss:2.0386827754339416 Accuracy:0.359375\n",
      "Loss:2.037789651304503 Accuracy:0.421875\n",
      "Loss:2.088724852693413 Accuracy:0.3671875\n",
      "Loss:2.0850830668552507 Accuracy:0.328125\n",
      "Loss:1.8893199690632803 Accuracy:0.3203125\n",
      "Loss:1.916868674142186 Accuracy:0.3671875\n",
      "Loss:2.129616612851616 Accuracy:0.375\n",
      "Loss:1.830401335266739 Accuracy:0.3671875\n",
      "Loss:2.0544375806335324 Accuracy:0.3125\n",
      "Loss:1.872719561329628 Accuracy:0.359375\n",
      "Loss:2.2291669284934565 Accuracy:0.2890625\n",
      "Loss:2.2614456271102386 Accuracy:0.2890625\n",
      "Loss:2.131931858558219 Accuracy:0.296875\n",
      "Loss:2.102914812968122 Accuracy:0.3125\n",
      "Loss:2.066882548556865 Accuracy:0.2890625\n",
      "Loss:1.9708477823195636 Accuracy:0.3671875\n",
      "Loss:1.7646530774227849 Accuracy:0.4140625\n",
      "Loss:1.9907889537785945 Accuracy:0.390625\n",
      "Loss:2.056302235206394 Accuracy:0.359375\n",
      "Loss:2.012885275209674 Accuracy:0.40625\n",
      "Loss:2.287954653647332 Accuracy:0.3046875\n",
      "Loss:1.8741612363737838 Accuracy:0.3828125\n",
      "Loss:1.9827454137294414 Accuracy:0.3203125\n",
      "Loss:2.045012886062346 Accuracy:0.3203125\n",
      "Loss:1.8936030732689306 Accuracy:0.34375\n",
      "Loss:2.108598968400024 Accuracy:0.3671875\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss:1.810505484430808 Accuracy:0.3828125\n",
      "Loss:1.8211349484930668 Accuracy:0.3984375\n",
      "Loss:2.1724315260590146 Accuracy:0.296875\n",
      "Loss:2.147736342990471 Accuracy:0.3203125\n",
      "Loss:1.8050105805560313 Accuracy:0.421875\n",
      "Loss:2.0601251296090752 Accuracy:0.328125\n",
      "Loss:2.1784448715061737 Accuracy:0.296875\n",
      "Loss:1.843428102034657 Accuracy:0.375\n",
      "Loss:1.6939369261697055 Accuracy:0.4609375\n",
      "Loss:2.0147881507027257 Accuracy:0.3359375\n",
      "Loss:1.8814305248249696 Accuracy:0.3671875\n",
      "Loss:1.804424487108216 Accuracy:0.3828125\n",
      "Loss:2.1010681093238315 Accuracy:0.3125\n",
      "Loss:1.8921658905338967 Accuracy:0.3515625\n",
      "Loss:1.9603157534443096 Accuracy:0.3359375\n",
      "Loss:1.7458286182935572 Accuracy:0.453125\n",
      "Loss:2.0782283572265157 Accuracy:0.2890625\n",
      "Loss:1.8799767426765912 Accuracy:0.3515625\n",
      "Loss:2.053086068642388 Accuracy:0.3125\n",
      "Loss:1.9127435582550059 Accuracy:0.34375\n",
      "Loss:1.9285762039533423 Accuracy:0.3515625\n",
      "Loss:1.953855284922849 Accuracy:0.3125\n",
      "Loss:1.9537273787508367 Accuracy:0.375\n",
      "Loss:1.8350654393524568 Accuracy:0.40625\n",
      "Loss:2.0943879738773825 Accuracy:0.34375\n",
      "Loss:1.9054937039646498 Accuracy:0.34375\n",
      "Loss:1.994274440623738 Accuracy:0.3046875\n",
      "Loss:1.8412375064794055 Accuracy:0.390625\n",
      "Loss:1.9267729246935277 Accuracy:0.3515625\n",
      "Loss:2.0001861470223137 Accuracy:0.3125\n",
      "Loss:1.883992131116827 Accuracy:0.3671875\n",
      "Loss:1.9074643005446075 Accuracy:0.3515625\n",
      "Loss:2.1265667422718675 Accuracy:0.328125\n",
      "Loss:1.7526021411102435 Accuracy:0.375\n",
      "Loss:1.904790860946511 Accuracy:0.3046875\n",
      "Loss:1.8519342595292128 Accuracy:0.3203125\n",
      "Loss:1.8057675906595383 Accuracy:0.421875\n",
      "Loss:2.0859976135125153 Accuracy:0.359375\n",
      "Loss:1.770462542558986 Accuracy:0.3515625\n",
      "Loss:1.6781988662778653 Accuracy:0.3984375\n",
      "Loss:1.746877918036658 Accuracy:0.421875\n",
      "Loss:1.99687569970638 Accuracy:0.34375\n",
      "Loss:1.7190844644220633 Accuracy:0.421875\n",
      "Loss:1.802913338991948 Accuracy:0.375\n",
      "Loss:1.891703624670093 Accuracy:0.359375\n",
      "Loss:1.7269247444809082 Accuracy:0.40625\n",
      "Loss:1.766524944574187 Accuracy:0.3671875\n",
      "Loss:1.9904273445794387 Accuracy:0.3125\n",
      "Loss:1.8500615121938715 Accuracy:0.40625\n",
      "Loss:1.7383766464289856 Accuracy:0.4296875\n",
      "Loss:1.7642366768467688 Accuracy:0.3671875\n",
      "Loss:1.7888038646979818 Accuracy:0.4765625\n",
      "Loss:1.760665019654157 Accuracy:0.375\n",
      "Loss:1.728902175941664 Accuracy:0.359375\n",
      "Loss:1.9488168791604075 Accuracy:0.375\n",
      "Loss:1.6644763320955223 Accuracy:0.4453125\n",
      "Loss:1.8824849602455953 Accuracy:0.3828125\n",
      "Loss:1.7552275688920642 Accuracy:0.4140625\n",
      "Loss:1.670128440808595 Accuracy:0.4375\n",
      "Loss:1.6849401909828416 Accuracy:0.390625\n",
      "Loss:1.6949463622682646 Accuracy:0.453125\n",
      "Loss:1.9988007600186828 Accuracy:0.3125\n",
      "Loss:1.845091296502265 Accuracy:0.390625\n",
      "Loss:2.1683433504325444 Accuracy:0.328125\n",
      "Loss:2.027464924566907 Accuracy:0.359375\n",
      "Loss:1.817284172662374 Accuracy:0.3671875\n",
      "Loss:1.7594087404232566 Accuracy:0.390625\n",
      "Loss:2.0535465171102816 Accuracy:0.328125\n",
      "Loss:1.7851877074758904 Accuracy:0.375\n",
      "Loss:1.9438288492484121 Accuracy:0.328125\n",
      "Loss:2.0469199742531767 Accuracy:0.3203125\n",
      "Loss:1.7090655997477964 Accuracy:0.3515625\n",
      "Loss:1.8143381079007737 Accuracy:0.3671875\n",
      "Loss:1.705370289022417 Accuracy:0.390625\n",
      "Loss:1.8799065416942808 Accuracy:0.3671875\n",
      "Loss:1.8138967601376086 Accuracy:0.40625\n",
      "Loss:2.0268845157309716 Accuracy:0.34375\n",
      "Loss:1.7100552554491721 Accuracy:0.3828125\n",
      "Loss:1.935522093285159 Accuracy:0.3125\n",
      "Loss:1.8685307231647221 Accuracy:0.40625\n",
      "Loss:1.8252151875652312 Accuracy:0.3984375\n",
      "Loss:1.7625788372926507 Accuracy:0.3828125\n",
      "Loss:1.6906683198979677 Accuracy:0.453125\n",
      "Loss:1.5109669435269466 Accuracy:0.4609375\n",
      "Loss:1.8234238733563708 Accuracy:0.421875\n",
      "Loss:1.850151823151134 Accuracy:0.3515625\n",
      "Loss:1.7468306216184626 Accuracy:0.375\n",
      "Loss:1.8498449024117598 Accuracy:0.359375\n",
      "Loss:1.8096159488553227 Accuracy:0.390625\n",
      "Loss:1.9166726683818593 Accuracy:0.359375\n",
      "Loss:1.9986840374036652 Accuracy:0.3671875\n",
      "Loss:1.9155545079971126 Accuracy:0.328125\n",
      "Loss:1.8080111619994303 Accuracy:0.4140625\n",
      "Loss:1.745535218289294 Accuracy:0.4609375\n",
      "Loss:1.7738721921141563 Accuracy:0.390625\n",
      "Loss:1.7854278373828418 Accuracy:0.359375\n",
      "Loss:1.8738418574385076 Accuracy:0.359375\n",
      "Loss:1.7555135635021892 Accuracy:0.359375\n",
      "Loss:1.8922702853176276 Accuracy:0.3359375\n",
      "Loss:2.160106865317978 Accuracy:0.390625\n",
      "Loss:1.8350770807037893 Accuracy:0.390625\n",
      "Loss:1.9989470817892767 Accuracy:0.3359375\n",
      "Loss:1.8317321370101016 Accuracy:0.3515625\n",
      "Loss:1.6184744109190863 Accuracy:0.3984375\n",
      "Loss:1.7888300932688361 Accuracy:0.3359375\n",
      "Loss:1.7951291727773706 Accuracy:0.359375\n",
      "Loss:1.9453939174538748 Accuracy:0.390625\n",
      "Loss:1.7861299763617502 Accuracy:0.3359375\n",
      "Loss:1.7022120542065686 Accuracy:0.3828125\n",
      "Loss:1.8185334128762893 Accuracy:0.4609375\n",
      "Loss:1.8681990747528059 Accuracy:0.34375\n",
      "Loss:1.5600334079655733 Accuracy:0.4296875\n",
      "Loss:1.923682196423239 Accuracy:0.3359375\n",
      "Loss:1.9646677953320346 Accuracy:0.3515625\n",
      "Loss:1.8753442167192524 Accuracy:0.359375\n",
      "Loss:1.7903745626526477 Accuracy:0.4140625\n",
      "Loss:2.0009305971929723 Accuracy:0.3671875\n",
      "Loss:1.8411824792610396 Accuracy:0.40625\n",
      "Loss:1.7305299544780097 Accuracy:0.3828125\n",
      "Loss:1.7493492044190613 Accuracy:0.3828125\n",
      "Loss:1.8594578924271041 Accuracy:0.3671875\n",
      "Loss:1.808461035990008 Accuracy:0.3671875\n",
      "Loss:1.7220395220278748 Accuracy:0.3359375\n",
      "Loss:2.026988188005575 Accuracy:0.328125\n",
      "Loss:1.7794418379283279 Accuracy:0.4140625\n",
      "Loss:1.89865821211257 Accuracy:0.328125\n",
      "Loss:1.7278109917283468 Accuracy:0.46875\n",
      "Loss:1.8490085428185528 Accuracy:0.3671875\n",
      "Loss:1.9369573845192154 Accuracy:0.3828125\n",
      "Loss:1.5888073024699634 Accuracy:0.4453125\n",
      "Loss:1.8330888901925513 Accuracy:0.40625\n",
      "Loss:1.7240335454567381 Accuracy:0.453125\n",
      "Loss:2.034474914813746 Accuracy:0.359375\n",
      "Loss:1.810564374024581 Accuracy:0.390625\n",
      "Loss:1.7915224461597539 Accuracy:0.359375\n",
      "Loss:1.811323688273057 Accuracy:0.40625\n",
      "Loss:1.7684920693379051 Accuracy:0.3515625\n",
      "Loss:1.9289570673618288 Accuracy:0.359375\n",
      "Loss:1.7322158754531378 Accuracy:0.3828125\n",
      "Loss:2.089573344502178 Accuracy:0.3046875\n",
      "Loss:1.7102985044471617 Accuracy:0.4296875\n",
      "Loss:1.5589100419204796 Accuracy:0.40625\n",
      "Loss:1.6030063140989128 Accuracy:0.40625\n",
      "Loss:1.951829017454041 Accuracy:0.359375\n",
      "Loss:1.667069016126669 Accuracy:0.421875\n",
      "Loss:1.8018702817252659 Accuracy:0.3828125\n",
      "Loss:1.7835334913680496 Accuracy:0.375\n",
      "Loss:1.4715547489740313 Accuracy:0.53125\n",
      "Loss:1.8448880848011702 Accuracy:0.4140625\n",
      "Loss:1.762085726403885 Accuracy:0.40625\n",
      "Loss:1.9084934476384472 Accuracy:0.359375\n",
      "Loss:1.7105713559277027 Accuracy:0.40625\n",
      "Loss:1.5875541054369042 Accuracy:0.4609375\n",
      "Loss:1.8496082111846712 Accuracy:0.3203125\n",
      "Loss:1.8791964996203232 Accuracy:0.359375\n",
      "Loss:1.7463970629355323 Accuracy:0.359375\n",
      "Loss:1.7348946714663953 Accuracy:0.3984375\n",
      "Loss:1.712935078562495 Accuracy:0.3984375\n",
      "Loss:1.7394609748742194 Accuracy:0.359375\n",
      "Loss:1.789578243508501 Accuracy:0.3671875\n",
      "Loss:1.6384570849216242 Accuracy:0.3828125\n",
      "Loss:1.8725445040589663 Accuracy:0.390625\n",
      "Loss:1.6682870908273078 Accuracy:0.3984375\n",
      "Loss:1.9359893909291492 Accuracy:0.421875\n",
      "Loss:1.7786332433051628 Accuracy:0.359375\n",
      "Loss:1.9228874532818687 Accuracy:0.421875\n",
      "Loss:1.610413618079387 Accuracy:0.4921875\n",
      "Loss:1.911766728269445 Accuracy:0.421875\n",
      "Loss:1.5739544489970396 Accuracy:0.40625\n",
      "Loss:1.6508823060961655 Accuracy:0.4765625\n",
      "Loss:1.8522300039790762 Accuracy:0.390625\n",
      "Loss:2.071477091925635 Accuracy:0.28125\n",
      "Loss:1.9579457658016213 Accuracy:0.390625\n",
      "Loss:2.0792750955011194 Accuracy:0.3125\n",
      "Loss:1.8411820940756578 Accuracy:0.375\n",
      "Loss:1.8407659054378982 Accuracy:0.375\n",
      "Loss:1.728101257794643 Accuracy:0.4296875\n",
      "Loss:1.80124063871166 Accuracy:0.390625\n",
      "Loss:1.9477776809033664 Accuracy:0.3359375\n",
      "Loss:1.806712462758589 Accuracy:0.3828125\n",
      "Loss:1.7772063392634598 Accuracy:0.375\n",
      "Loss:1.7118828009500195 Accuracy:0.3828125\n",
      "Loss:1.75374051778499 Accuracy:0.40625\n",
      "Loss:1.6602185815656303 Accuracy:0.4140625\n",
      "Loss:1.6822236066702776 Accuracy:0.46875\n",
      "Loss:1.5632142988973026 Accuracy:0.4453125\n",
      "Loss:1.9019446072898798 Accuracy:0.3515625\n",
      "Loss:1.8992109587773045 Accuracy:0.40625\n",
      "Loss:1.619741253031452 Accuracy:0.46875\n",
      "Loss:1.6093589921776115 Accuracy:0.4375\n",
      "Loss:1.8594180896552657 Accuracy:0.390625\n",
      "Loss:1.7972801721715437 Accuracy:0.4140625\n",
      "Loss:1.6702205786059068 Accuracy:0.421875\n",
      "Loss:1.9701972928755942 Accuracy:0.4140625\n",
      "Loss:1.7125120729250902 Accuracy:0.421875\n",
      "Loss:1.7574047202387717 Accuracy:0.3828125\n",
      "Loss:1.8571042071167192 Accuracy:0.4765625\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss:1.5806036257009408 Accuracy:0.46875\n",
      "Loss:1.6925283298530676 Accuracy:0.3828125\n",
      "Loss:1.7641192608628005 Accuracy:0.4375\n",
      "Loss:1.7753598880180954 Accuracy:0.3515625\n",
      "Loss:1.730286979799926 Accuracy:0.4140625\n",
      "Loss:1.581777973231863 Accuracy:0.4609375\n",
      "Loss:1.853185085757051 Accuracy:0.3828125\n",
      "Loss:1.6690690683480214 Accuracy:0.4375\n",
      "Loss:1.630236663497089 Accuracy:0.4375\n",
      "Loss:1.6242873310943062 Accuracy:0.4375\n",
      "Loss:1.5222046494479369 Accuracy:0.4375\n",
      "Loss:1.6170998259960538 Accuracy:0.4375\n",
      "Loss:1.5258794120236923 Accuracy:0.5\n",
      "Loss:1.662548471795121 Accuracy:0.390625\n",
      "Loss:1.668550669043106 Accuracy:0.4453125\n",
      "Loss:1.6780999486444177 Accuracy:0.4453125\n",
      "Loss:1.7769553030486533 Accuracy:0.40625\n",
      "Loss:1.4861128337924727 Accuracy:0.4609375\n",
      "Loss:1.7505983917008945 Accuracy:0.421875\n",
      "Loss:1.7513382483113704 Accuracy:0.3984375\n",
      "Loss:1.7687383013435023 Accuracy:0.3671875\n",
      "Loss:1.6866954318929954 Accuracy:0.4296875\n",
      "Loss:1.6450193672394364 Accuracy:0.40625\n",
      "Loss:1.7519442057832553 Accuracy:0.3984375\n",
      "Loss:1.7845676365658272 Accuracy:0.3828125\n",
      "Loss:1.6828775264893534 Accuracy:0.375\n",
      "Loss:1.6245591520020133 Accuracy:0.4453125\n",
      "Loss:1.7704604704748985 Accuracy:0.4375\n",
      "Loss:1.57303891800247 Accuracy:0.3984375\n",
      "Loss:1.5799350494944682 Accuracy:0.421875\n",
      "Loss:1.6239545336807215 Accuracy:0.4453125\n",
      "Loss:1.7141303949552276 Accuracy:0.4140625\n",
      "Loss:1.665662111682892 Accuracy:0.4140625\n",
      "Loss:1.7721908189085251 Accuracy:0.3828125\n",
      "Loss:1.6089849309641235 Accuracy:0.421875\n",
      "Loss:1.5516058785538123 Accuracy:0.46875\n",
      "Loss:1.5444828929100998 Accuracy:0.4453125\n",
      "Loss:1.5706112721740173 Accuracy:0.4140625\n",
      "Loss:1.4310244211608916 Accuracy:0.4765625\n",
      "Loss:1.7404643433577909 Accuracy:0.40625\n",
      "Loss:1.8187834598638541 Accuracy:0.3671875\n",
      "Loss:1.417808058485477 Accuracy:0.546875\n",
      "Loss:1.7191317497891336 Accuracy:0.4609375\n",
      "Loss:1.7046529749436625 Accuracy:0.453125\n",
      "Loss:1.4911541730616253 Accuracy:0.4765625\n",
      "Loss:1.5952534498599695 Accuracy:0.4765625\n",
      "Loss:1.6758819487772887 Accuracy:0.40625\n",
      "Loss:1.658397111032624 Accuracy:0.421875\n",
      "Loss:1.8216116671554397 Accuracy:0.390625\n",
      "Loss:1.7908291816462125 Accuracy:0.3828125\n",
      "Loss:1.745579216671912 Accuracy:0.3828125\n",
      "Loss:1.4700461522841377 Accuracy:0.4453125\n",
      "Loss:1.7458914706461157 Accuracy:0.40625\n",
      "Loss:1.6429990796549139 Accuracy:0.40625\n",
      "Loss:1.9527105273200327 Accuracy:0.34375\n",
      "Loss:1.7315322434734872 Accuracy:0.453125\n",
      "Loss:1.5532662654539044 Accuracy:0.484375\n",
      "Loss:1.6260328612582806 Accuracy:0.40625\n",
      "Loss:1.9012458544950628 Accuracy:0.390625\n",
      "Loss:1.6374519937973293 Accuracy:0.40625\n",
      "Loss:1.5235706154645492 Accuracy:0.4921875\n",
      "Loss:1.7561955150987782 Accuracy:0.46875\n",
      "Loss:1.534771476189324 Accuracy:0.421875\n",
      "Loss:1.6639814445573502 Accuracy:0.3984375\n",
      "Loss:1.75572417292419 Accuracy:0.421875\n",
      "Loss:1.5596482825601239 Accuracy:0.3984375\n",
      "Loss:1.6148889257609984 Accuracy:0.453125\n",
      "Loss:1.6308260905491392 Accuracy:0.4140625\n",
      "Loss:1.8796079012447526 Accuracy:0.40625\n",
      "Loss:1.8090989120572423 Accuracy:0.421875\n",
      "Loss:1.7447271204457446 Accuracy:0.390625\n",
      "Loss:1.6146080795052753 Accuracy:0.375\n",
      "Loss:1.5226171881880979 Accuracy:0.46875\n",
      "Loss:1.4949219336566588 Accuracy:0.46875\n",
      "Loss:1.6665925812629965 Accuracy:0.3828125\n",
      "Loss:1.723474237162931 Accuracy:0.4296875\n",
      "Loss:1.822008999184231 Accuracy:0.3125\n",
      "Loss:1.8185669939978462 Accuracy:0.390625\n",
      "Loss:1.6916480317233669 Accuracy:0.3984375\n",
      "Loss:1.7005113189404526 Accuracy:0.390625\n",
      "Loss:1.7907207013286128 Accuracy:0.359375\n",
      "Loss:1.584089998667927 Accuracy:0.4453125\n",
      "Loss:1.7383991600500732 Accuracy:0.4140625\n",
      "Loss:1.4980458390554037 Accuracy:0.4609375\n",
      "Loss:1.5978624127655645 Accuracy:0.5\n",
      "Loss:1.7843040437865207 Accuracy:0.40625\n",
      "Loss:1.929730628035812 Accuracy:0.359375\n",
      "Loss:1.4330356376464453 Accuracy:0.46875\n",
      "Loss:1.6737891977747483 Accuracy:0.390625\n",
      "Loss:1.4412289416938404 Accuracy:0.4453125\n",
      "Loss:1.6045141047023521 Accuracy:0.4921875\n",
      "Loss:1.5537431670845372 Accuracy:0.4765625\n",
      "Loss:1.6152053988961508 Accuracy:0.4375\n",
      "Loss:1.8121539775350604 Accuracy:0.3671875\n",
      "Loss:1.7195432601170513 Accuracy:0.40625\n",
      "Loss:1.8066091218050107 Accuracy:0.4140625\n",
      "Loss:1.8496023383987563 Accuracy:0.3828125\n",
      "Loss:1.6486875783439872 Accuracy:0.4140625\n",
      "Loss:1.6037050206040389 Accuracy:0.421875\n",
      "Loss:1.7690947150951752 Accuracy:0.4140625\n",
      "Loss:1.7808132627302293 Accuracy:0.40625\n",
      "Loss:1.556267030820646 Accuracy:0.390625\n",
      "Loss:1.7996717898628325 Accuracy:0.375\n",
      "Loss:1.621085434137499 Accuracy:0.4375\n",
      "Loss:1.5061851600511447 Accuracy:0.4375\n",
      "Loss:1.5676428148756947 Accuracy:0.4453125\n",
      "Loss:1.7186701336891606 Accuracy:0.34375\n",
      "Loss:1.7750175019362895 Accuracy:0.4453125\n",
      "Loss:1.7842430504015838 Accuracy:0.4140625\n",
      "Loss:1.7898470762716432 Accuracy:0.4765625\n",
      "Loss:1.8082544071737139 Accuracy:0.4296875\n",
      "Loss:1.792146081763085 Accuracy:0.359375\n",
      "Loss:1.6437896128359313 Accuracy:0.4453125\n",
      "Loss:1.724391283400483 Accuracy:0.40625\n",
      "Loss:1.6931605554384133 Accuracy:0.4296875\n",
      "Loss:1.7061066746828923 Accuracy:0.359375\n",
      "Loss:1.6414726942486797 Accuracy:0.4453125\n",
      "Loss:1.682584231570858 Accuracy:0.46875\n",
      "Loss:1.7227585990322438 Accuracy:0.4296875\n",
      "Loss:1.6878445945399783 Accuracy:0.4765625\n",
      "Loss:1.3769412095613722 Accuracy:0.46875\n",
      "Loss:1.6011059961601573 Accuracy:0.421875\n",
      "Loss:1.6321226033078293 Accuracy:0.4765625\n",
      "Loss:1.4641751698374748 Accuracy:0.46875\n",
      "Loss:1.627880417459194 Accuracy:0.40625\n",
      "Loss:1.5668989330223728 Accuracy:0.4140625\n",
      "Loss:1.548473215911316 Accuracy:0.46875\n",
      "Loss:1.4848337870859285 Accuracy:0.4609375\n",
      "Loss:1.486247758677078 Accuracy:0.4609375\n",
      "Loss:1.7532778025075386 Accuracy:0.4296875\n",
      "Loss:1.8419022624402526 Accuracy:0.328125\n",
      "Loss:1.520850410432996 Accuracy:0.4375\n",
      "Loss:1.5903624090381017 Accuracy:0.4609375\n",
      "Loss:1.4455045062135305 Accuracy:0.515625\n",
      "Loss:1.7627960830335594 Accuracy:0.375\n",
      "Loss:1.564742404392428 Accuracy:0.5078125\n",
      "Loss:1.615694202846123 Accuracy:0.4453125\n",
      "Loss:1.6022723510705799 Accuracy:0.3828125\n",
      "Loss:1.6092931854414183 Accuracy:0.4375\n",
      "Loss:1.7890796922034182 Accuracy:0.4296875\n",
      "Loss:1.8661186577161875 Accuracy:0.3359375\n",
      "Loss:1.8370541539081795 Accuracy:0.421875\n",
      "Loss:1.6738385918510876 Accuracy:0.390625\n",
      "Loss:1.8001812677873503 Accuracy:0.390625\n",
      "Loss:1.70104363876061 Accuracy:0.453125\n",
      "Loss:1.5096467118168158 Accuracy:0.4296875\n",
      "Loss:1.5696707335757263 Accuracy:0.46875\n",
      "Loss:1.6656593507162032 Accuracy:0.4296875\n",
      "Loss:1.487547288913726 Accuracy:0.4921875\n",
      "Loss:1.5103224203231922 Accuracy:0.484375\n",
      "Loss:1.695272823768832 Accuracy:0.40625\n",
      "Loss:1.6461196297106329 Accuracy:0.3984375\n",
      "Loss:1.7354793103696133 Accuracy:0.359375\n",
      "Loss:1.785708244726119 Accuracy:0.375\n",
      "Loss:1.4156818711228505 Accuracy:0.5390625\n",
      "Loss:1.486470680887085 Accuracy:0.4140625\n",
      "Loss:1.5832526686681427 Accuracy:0.4375\n",
      "Loss:1.453687191887739 Accuracy:0.4921875\n",
      "Loss:1.721080143724373 Accuracy:0.390625\n",
      "Loss:1.7604925647283949 Accuracy:0.3984375\n",
      "Loss:1.5829450191912842 Accuracy:0.4765625\n",
      "Loss:1.6156356428498855 Accuracy:0.4453125\n",
      "Loss:1.5852953580325195 Accuracy:0.4140625\n",
      "Loss:1.672659914830944 Accuracy:0.4375\n",
      "Loss:1.539388852660722 Accuracy:0.453125\n",
      "Loss:1.7680824265401847 Accuracy:0.40625\n",
      "Loss:1.636228172666026 Accuracy:0.484375\n",
      "Loss:1.8621202224496238 Accuracy:0.3515625\n",
      "Loss:1.575871672886465 Accuracy:0.46875\n",
      "Loss:1.4885878312465652 Accuracy:0.5\n",
      "Loss:1.700843806994803 Accuracy:0.4296875\n",
      "Loss:1.6303110639563299 Accuracy:0.4453125\n",
      "Loss:1.6204145349609411 Accuracy:0.4296875\n",
      "Loss:1.535467680253816 Accuracy:0.46875\n",
      "Loss:1.642372731128267 Accuracy:0.390625\n",
      "Loss:1.8599407073719985 Accuracy:0.3828125\n",
      "Loss:1.5787617149385529 Accuracy:0.40625\n",
      "Loss:1.5704591539471235 Accuracy:0.46875\n",
      "Loss:1.5047933020423976 Accuracy:0.4765625\n",
      "Loss:1.5726087381025993 Accuracy:0.4375\n",
      "Loss:1.7049068928264237 Accuracy:0.359375\n",
      "Loss:1.5762852870010742 Accuracy:0.375\n",
      "Loss:1.6126198643560037 Accuracy:0.3671875\n",
      "Loss:1.602856207345567 Accuracy:0.4765625\n",
      "Loss:1.5420156371853833 Accuracy:0.4453125\n",
      "Loss:1.5775169345488909 Accuracy:0.453125\n",
      "Loss:1.4533428612287176 Accuracy:0.5390625\n",
      "Loss:1.5566559763922982 Accuracy:0.453125\n",
      "Loss:1.6892359670821837 Accuracy:0.390625\n",
      "Loss:1.588534036376696 Accuracy:0.4765625\n",
      "Loss:1.44288676121647 Accuracy:0.484375\n",
      "Loss:1.6643598316244583 Accuracy:0.3828125\n",
      "Loss:1.6227073710397737 Accuracy:0.40625\n",
      "Loss:1.7599188874805496 Accuracy:0.3984375\n",
      "Loss:1.6431867243559224 Accuracy:0.46875\n",
      "Loss:1.6810494540477794 Accuracy:0.46875\n",
      "Loss:1.5864975298304045 Accuracy:0.4765625\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss:1.7210548892518025 Accuracy:0.40625\n",
      "Loss:1.7877002626869334 Accuracy:0.3359375\n",
      "Loss:1.8476971005882852 Accuracy:0.375\n",
      "Loss:1.59878151495421 Accuracy:0.421875\n",
      "Loss:1.6436037239794836 Accuracy:0.40625\n",
      "Loss:1.6370805734696163 Accuracy:0.421875\n",
      "Loss:1.5871349381356774 Accuracy:0.453125\n",
      "Loss:1.6460562129397873 Accuracy:0.4296875\n",
      "Loss:1.6832093429378108 Accuracy:0.4140625\n",
      "Loss:1.6265367977213812 Accuracy:0.375\n",
      "Loss:1.715028522549252 Accuracy:0.4375\n",
      "Loss:1.5609003697741146 Accuracy:0.4765625\n",
      "Loss:2.0145958791263894 Accuracy:0.34375\n",
      "Loss:1.8706389614285568 Accuracy:0.3046875\n",
      "Loss:1.730468046398067 Accuracy:0.375\n",
      "Loss:1.7034336544263917 Accuracy:0.40625\n",
      "Loss:1.5088377620876816 Accuracy:0.46875\n",
      "Loss:1.6254970428194029 Accuracy:0.4609375\n",
      "Loss:1.4775088569304695 Accuracy:0.4609375\n",
      "Loss:1.8173168330096108 Accuracy:0.3671875\n",
      "Loss:1.8573754542034546 Accuracy:0.34375\n",
      "Loss:1.6352236187465432 Accuracy:0.453125\n",
      "Loss:1.6418298978796635 Accuracy:0.4609375\n",
      "Loss:1.6400396041730891 Accuracy:0.4140625\n",
      "Loss:1.7333636790569118 Accuracy:0.3984375\n",
      "Loss:1.8303040432178461 Accuracy:0.40625\n",
      "Loss:1.5737793513397154 Accuracy:0.421875\n",
      "Loss:1.7602580385711222 Accuracy:0.4140625\n",
      "Loss:1.8423238116654734 Accuracy:0.390625\n",
      "Loss:1.5265844905083958 Accuracy:0.4296875\n",
      "Loss:1.6191890759477414 Accuracy:0.4609375\n",
      "Loss:1.4920021924171416 Accuracy:0.453125\n",
      "Loss:1.8702210905959715 Accuracy:0.3671875\n",
      "Loss:1.63411170673313 Accuracy:0.375\n",
      "Loss:1.5375412343780073 Accuracy:0.4296875\n",
      "Loss:1.5865861262489291 Accuracy:0.4453125\n",
      "Loss:1.5767321183502425 Accuracy:0.390625\n",
      "Loss:1.733411088462121 Accuracy:0.375\n",
      "Loss:1.6613087951158505 Accuracy:0.375\n",
      "Loss:1.6980189480790586 Accuracy:0.4375\n",
      "Loss:1.5737181907307956 Accuracy:0.40625\n",
      "Loss:1.4852032795564787 Accuracy:0.4921875\n",
      "Loss:1.685767889837293 Accuracy:0.4375\n",
      "Loss:1.71691556274715 Accuracy:0.4296875\n",
      "Loss:1.5638460590555598 Accuracy:0.4609375\n",
      "Loss:1.7068500084228981 Accuracy:0.390625\n",
      "Loss:1.5904028754497719 Accuracy:0.4375\n",
      "Loss:1.5988424037664783 Accuracy:0.46875\n",
      "Loss:1.6987682664148078 Accuracy:0.453125\n",
      "Loss:1.78299064099023 Accuracy:0.390625\n",
      "Loss:1.6061474948228303 Accuracy:0.453125\n",
      "Loss:1.520428678383549 Accuracy:0.4765625\n",
      "Loss:1.4656229155705218 Accuracy:0.4921875\n",
      "Loss:1.538825132307449 Accuracy:0.4609375\n",
      "Loss:1.5442688523250254 Accuracy:0.4140625\n",
      "Loss:1.4366432495260437 Accuracy:0.484375\n",
      "Loss:1.4983022944160238 Accuracy:0.46875\n",
      "Loss:1.2486703592118245 Accuracy:0.5078125\n",
      "Loss:1.4900841361754797 Accuracy:0.4609375\n",
      "Loss:1.566269002528434 Accuracy:0.4375\n",
      "Loss:1.6569218718075762 Accuracy:0.421875\n",
      "Loss:1.474839070545844 Accuracy:0.484375\n",
      "Loss:1.5336765464042241 Accuracy:0.421875\n",
      "Loss:1.6989862483292462 Accuracy:0.3671875\n",
      "Loss:1.5247009405286622 Accuracy:0.4375\n",
      "Loss:1.4630576550131984 Accuracy:0.515625\n",
      "Loss:1.5204710601253768 Accuracy:0.46875\n",
      "Loss:1.7015673011304575 Accuracy:0.3984375\n",
      "Loss:1.7285803348363364 Accuracy:0.3515625\n",
      "Loss:1.4116892556009628 Accuracy:0.484375\n",
      "Loss:1.5367095925796754 Accuracy:0.4453125\n",
      "Loss:1.608440091144005 Accuracy:0.3984375\n",
      "Loss:1.6347951662729736 Accuracy:0.4375\n",
      "Loss:1.8130631522606762 Accuracy:0.3203125\n",
      "Loss:1.5544747950569113 Accuracy:0.46875\n",
      "Loss:1.718955609413824 Accuracy:0.3984375\n",
      "Loss:1.6707452412529853 Accuracy:0.40625\n",
      "Loss:1.7284246308136801 Accuracy:0.3203125\n",
      "Loss:1.561919830797985 Accuracy:0.4140625\n",
      "Loss:1.637056964242416 Accuracy:0.375\n",
      "Loss:1.439576922585542 Accuracy:0.4453125\n",
      "Loss:1.7258563409823062 Accuracy:0.390625\n",
      "Loss:1.4437818007020313 Accuracy:0.484375\n",
      "Loss:1.6893748640635768 Accuracy:0.40625\n",
      "Loss:1.586975800836157 Accuracy:0.4140625\n",
      "Loss:1.7875479800575853 Accuracy:0.390625\n",
      "Loss:1.615119629879701 Accuracy:0.4296875\n",
      "Loss:1.7043553699591816 Accuracy:0.46875\n",
      "Loss:1.633674197475 Accuracy:0.4140625\n",
      "Loss:1.572446953304294 Accuracy:0.4296875\n",
      "Loss:1.672285953195189 Accuracy:0.4765625\n",
      "Loss:1.5794620899053562 Accuracy:0.4296875\n",
      "Loss:1.527028216034727 Accuracy:0.4609375\n",
      "Loss:1.5874484839373513 Accuracy:0.4375\n",
      "Loss:1.5775730094645355 Accuracy:0.40625\n",
      "Loss:1.7246408191853817 Accuracy:0.4921875\n",
      "Loss:1.5960233450276 Accuracy:0.4375\n",
      "Loss:1.5080264435406372 Accuracy:0.4296875\n",
      "Loss:1.5910355964656375 Accuracy:0.4609375\n",
      "Loss:1.682016736136985 Accuracy:0.4140625\n",
      "Loss:1.495358974746716 Accuracy:0.46875\n",
      "Loss:1.6750156560902125 Accuracy:0.4609375\n",
      "Loss:1.5937098775052694 Accuracy:0.4296875\n",
      "Loss:1.6034746608274943 Accuracy:0.421875\n",
      "Loss:1.6044678815220808 Accuracy:0.4296875\n",
      "Loss:1.6155557352995942 Accuracy:0.4296875\n",
      "Loss:1.657461869868702 Accuracy:0.4296875\n",
      "Loss:1.5833902088719949 Accuracy:0.4296875\n",
      "Loss:1.461577227898532 Accuracy:0.4453125\n",
      "Loss:1.5265865909701426 Accuracy:0.4609375\n",
      "Loss:1.5831752917687032 Accuracy:0.4296875\n",
      "Loss:1.558388571866404 Accuracy:0.453125\n",
      "Loss:1.589273477411816 Accuracy:0.4296875\n",
      "Loss:1.7120323250219442 Accuracy:0.4453125\n",
      "Loss:1.6862450710160455 Accuracy:0.40625\n",
      "Loss:1.476459778647678 Accuracy:0.4296875\n",
      "Loss:1.5560331103359457 Accuracy:0.453125\n",
      "Loss:1.6564916859963827 Accuracy:0.375\n",
      "Loss:1.4757276977777238 Accuracy:0.4375\n",
      "Loss:1.5138715129618419 Accuracy:0.53125\n",
      "Loss:1.6360641613558793 Accuracy:0.421875\n",
      "Loss:1.480514934929181 Accuracy:0.4609375\n",
      "Loss:1.6122451413554169 Accuracy:0.453125\n",
      "Loss:1.5830528887939284 Accuracy:0.4453125\n",
      "Loss:1.688147545777932 Accuracy:0.421875\n",
      "Loss:1.676299782005401 Accuracy:0.46875\n",
      "Loss:1.6737420616854481 Accuracy:0.4140625\n",
      "Loss:1.7128703006830588 Accuracy:0.390625\n",
      "Loss:1.785557595433719 Accuracy:0.3515625\n",
      "Loss:1.375579418148002 Accuracy:0.5078125\n",
      "Loss:1.457923256974661 Accuracy:0.4609375\n",
      "Loss:1.7686568997057894 Accuracy:0.4140625\n",
      "Loss:1.435610635450295 Accuracy:0.4453125\n",
      "Loss:1.7952392497716274 Accuracy:0.421875\n",
      "Loss:1.584081995491469 Accuracy:0.4453125\n",
      "Loss:1.6664068100412628 Accuracy:0.4375\n",
      "Loss:1.571755115295453 Accuracy:0.3984375\n",
      "Loss:1.5850684587311674 Accuracy:0.40625\n",
      "Loss:1.6530459771706028 Accuracy:0.421875\n",
      "Loss:1.537155381060299 Accuracy:0.5\n",
      "Loss:1.8819303263783527 Accuracy:0.3828125\n",
      "Loss:1.5744591437372235 Accuracy:0.484375\n",
      "Loss:1.4754047509372195 Accuracy:0.515625\n",
      "Loss:1.6017304222638487 Accuracy:0.421875\n",
      "Loss:1.6524491341513259 Accuracy:0.4609375\n",
      "Loss:1.3872905463074394 Accuracy:0.5234375\n",
      "Loss:1.4496069347817806 Accuracy:0.4765625\n",
      "Loss:1.7251008691342948 Accuracy:0.34375\n",
      "Loss:1.6059219835933205 Accuracy:0.4609375\n",
      "Loss:1.7333858378901947 Accuracy:0.359375\n",
      "Loss:1.623993226766888 Accuracy:0.4296875\n",
      "Loss:1.5677783640521463 Accuracy:0.4296875\n",
      "Loss:1.6836573275496844 Accuracy:0.390625\n",
      "Loss:1.4395566369810273 Accuracy:0.453125\n",
      "Loss:1.6146928381908947 Accuracy:0.4609375\n",
      "Loss:1.6215509818973783 Accuracy:0.4453125\n",
      "Loss:1.8797427365593329 Accuracy:0.3671875\n",
      "Loss:1.6267112289781445 Accuracy:0.421875\n",
      "Loss:1.731983236548053 Accuracy:0.40625\n",
      "Loss:1.62134432228705 Accuracy:0.40625\n",
      "Loss:1.5799316805706722 Accuracy:0.375\n",
      "Loss:1.4742636039057508 Accuracy:0.46875\n",
      "Loss:1.509899785020953 Accuracy:0.453125\n",
      "Loss:1.7098659324451104 Accuracy:0.375\n",
      "Loss:1.762320945547059 Accuracy:0.3828125\n",
      "Loss:1.4514101774643462 Accuracy:0.4609375\n",
      "Loss:1.6797984632436602 Accuracy:0.3984375\n",
      "Loss:1.7047695360387638 Accuracy:0.4140625\n",
      "Loss:1.5239275063719107 Accuracy:0.453125\n",
      "Loss:1.66088345819738 Accuracy:0.390625\n",
      "Loss:1.4819603278209676 Accuracy:0.4609375\n",
      "Loss:1.6872847102853183 Accuracy:0.40625\n",
      "Loss:1.6756231994502513 Accuracy:0.390625\n",
      "Loss:1.5251695477674705 Accuracy:0.3984375\n",
      "Loss:1.5060017186590995 Accuracy:0.484375\n",
      "Loss:1.6494772763688772 Accuracy:0.4765625\n",
      "Loss:1.6155687885954753 Accuracy:0.46875\n",
      "Loss:1.6981302937526521 Accuracy:0.5\n",
      "Loss:1.6276687941399282 Accuracy:0.4765625\n",
      "Loss:1.4193098096997425 Accuracy:0.5\n",
      "Loss:1.6421678107643412 Accuracy:0.4609375\n",
      "Loss:1.5596335060203321 Accuracy:0.484375\n",
      "Loss:1.5540986619130694 Accuracy:0.5\n",
      "Loss:1.415218853160507 Accuracy:0.5234375\n",
      "Loss:1.5827567576996766 Accuracy:0.46875\n",
      "Loss:1.5337955128017287 Accuracy:0.4140625\n",
      "Loss:1.6637426361538283 Accuracy:0.3515625\n",
      "Loss:1.3913828191022852 Accuracy:0.4765625\n",
      "Loss:1.6548115041581866 Accuracy:0.4375\n",
      "Loss:1.5294791197679134 Accuracy:0.5\n",
      "Loss:1.6465430654458124 Accuracy:0.3984375\n",
      "Loss:1.8004509302551919 Accuracy:0.3671875\n",
      "Loss:1.5422037301479998 Accuracy:0.4453125\n",
      "Loss:1.6676568282294821 Accuracy:0.4140625\n",
      "Loss:1.3955879471410133 Accuracy:0.484375\n",
      "Loss:1.6449048534685669 Accuracy:0.4296875\n",
      "Loss:1.545713926728754 Accuracy:0.46875\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss:1.5512829753495336 Accuracy:0.5078125\n",
      "Loss:1.5479610458015256 Accuracy:0.4609375\n",
      "Loss:1.4907743180128128 Accuracy:0.5078125\n",
      "Loss:1.6504742420388308 Accuracy:0.4296875\n",
      "Loss:1.5498188932420964 Accuracy:0.484375\n",
      "Loss:1.6584083222000634 Accuracy:0.3671875\n",
      "Loss:1.4318784449520714 Accuracy:0.5\n",
      "Loss:1.8021580948787586 Accuracy:0.4140625\n",
      "Loss:1.7440417024990456 Accuracy:0.3828125\n",
      "Loss:1.4877314556719008 Accuracy:0.4296875\n",
      "Loss:1.5767535708093336 Accuracy:0.4296875\n",
      "Loss:1.8482584679267346 Accuracy:0.375\n",
      "Loss:1.6564232357304713 Accuracy:0.421875\n",
      "Loss:1.5715171158493086 Accuracy:0.4140625\n",
      "Loss:1.4512983142512612 Accuracy:0.515625\n",
      "Loss:1.5629199641067242 Accuracy:0.453125\n",
      "Loss:1.5082638884598092 Accuracy:0.453125\n",
      "Loss:1.4597975588881225 Accuracy:0.46875\n",
      "Loss:1.4149401668551325 Accuracy:0.4921875\n",
      "Loss:1.7922196932595678 Accuracy:0.40625\n",
      "Loss:1.5130836043098372 Accuracy:0.546875\n",
      "Loss:1.5240396093979898 Accuracy:0.4375\n",
      "Loss:1.5322911620642383 Accuracy:0.4765625\n",
      "Loss:1.7385814317774688 Accuracy:0.3671875\n",
      "Loss:1.6010540285795405 Accuracy:0.515625\n",
      "Loss:1.6408066853997165 Accuracy:0.5078125\n",
      "Loss:1.8284274826904818 Accuracy:0.3671875\n",
      "Loss:1.5405961781985247 Accuracy:0.484375\n",
      "Loss:1.553922786183083 Accuracy:0.4921875\n",
      "Loss:1.7688568103510178 Accuracy:0.4140625\n",
      "Loss:1.4557426220664689 Accuracy:0.4921875\n",
      "Loss:1.655133896941083 Accuracy:0.4296875\n",
      "Loss:1.7645202945082623 Accuracy:0.3984375\n",
      "Loss:1.7661305156311042 Accuracy:0.40625\n",
      "Loss:1.3432481474218974 Accuracy:0.5\n",
      "Loss:1.542634413866451 Accuracy:0.4296875\n",
      "Loss:1.8941872561349276 Accuracy:0.34375\n",
      "Loss:1.945660028783759 Accuracy:0.3515625\n",
      "Loss:1.584382721019601 Accuracy:0.4609375\n",
      "Loss:1.760756843604213 Accuracy:0.3671875\n",
      "Loss:1.6819917001639713 Accuracy:0.4453125\n",
      "Loss:1.536904412737579 Accuracy:0.4453125\n",
      "Loss:1.4758118083582636 Accuracy:0.4765625\n",
      "Loss:1.6867950452494875 Accuracy:0.40625\n",
      "Loss:1.8657716197143466 Accuracy:0.28125\n",
      "Loss:1.5979609629179916 Accuracy:0.453125\n",
      "Loss:1.581691747309579 Accuracy:0.3984375\n",
      "Loss:1.5056230454979653 Accuracy:0.4765625\n",
      "Loss:1.4268363203958305 Accuracy:0.4765625\n",
      "Loss:1.7402421979353413 Accuracy:0.4609375\n",
      "Loss:1.6249129846163655 Accuracy:0.3671875\n",
      "Loss:1.6558694965573821 Accuracy:0.4296875\n",
      "Loss:1.70551635829962 Accuracy:0.3984375\n"
     ]
    }
   ],
   "source": [
    "# Train mini batch\n",
    "np.random.seed(12)\n",
    "\n",
    "\n",
    "net = CNN(in_shape=X.shape[1:], out_size=Y.shape[1])\n",
    "criteria = Criteria(mode='cross_entropy')\n",
    "optimizer = Adam(net,learning_rate=0.01)\n",
    "\n",
    "\n",
    "for i in range(num_iter):\n",
    "    \n",
    "    permutation = np.random.permutation(m)\n",
    "    \n",
    "    for j in range(0,m,batch_size):\n",
    "        \n",
    "        indices = permutation[j:j+batch_size]\n",
    "        X_batch, Y_batch = X[indices], Y[indices]\n",
    "        \n",
    "        A = net.forward(X_batch)\n",
    "\n",
    "        loss = criteria.forward(A, Y_batch)\n",
    "        \n",
    "        dZ = criteria.backward()\n",
    "        dZ = net.backward(dZ)\n",
    "\n",
    "        optimizer.step(net)\n",
    "        \n",
    "        accuracy = np.sum(np.argmax(A,axis=1)==np.argmax(Y_batch,axis=1))/len(X_batch)\n",
    "        print(f\"Loss:{loss} Accuracy:{accuracy}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Num test:6000 Loss:1.5793956697163378 Accuracy:0.44716666666666666\n"
     ]
    }
   ],
   "source": [
    "A = net.forward(X_test)\n",
    "loss = criteria.forward(A, Y_test)\n",
    "accuracy = np.sum(np.argmax(A,axis=1)==np.argmax(Y_test,axis=1))/len(X_test)\n",
    "\n",
    "print(f\" Num test:{len(X_test)} Loss:{loss} Accuracy:{accuracy}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Better than fully connected model while using substaintialy smaller number of parameters."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}