fashion_mnist_classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Fashion MNIST Data Read\n",
    "\n",
    "use load_mnist function in https://github.com/zalandoresearch/fashion-mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Define Load MNIST\n",
    "\"\"\"\n",
    "import os\n",
    "import gzip\n",
    "import numpy as np\n",
    "\n",
    "def load_mnist(path, kind='train'):\n",
    "    \"\"\"Load MNIST data from `path`\"\"\"\n",
    "    labels_path = os.path.join(path,\n",
    "                               '%s-labels-idx1-ubyte.gz'\n",
    "                               % kind)\n",
    "    images_path = os.path.join(path,\n",
    "                               '%s-images-idx3-ubyte.gz'\n",
    "                               % kind)\n",
    "\n",
    "    with gzip.open(labels_path, 'rb') as lbpath:\n",
    "        labels = np.frombuffer(lbpath.read(), dtype=np.uint8,\n",
    "                               offset=8)\n",
    "\n",
    "    with gzip.open(images_path, 'rb') as imgpath:\n",
    "        images = np.frombuffer(imgpath.read(), dtype=np.uint8,\n",
    "                               offset=16).reshape(len(labels), 784)\n",
    "\n",
    "    return images, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Class 0 Number: 6000\n",
      "Class 1 Number: 6000\n",
      "Class 2 Number: 6000\n",
      "Class 3 Number: 6000\n",
      "Class 4 Number: 6000\n",
      "Class 5 Number: 6000\n",
      "Class 6 Number: 6000\n",
      "Class 7 Number: 6000\n",
      "Class 8 Number: 6000\n",
      "Class 9 Number: 6000\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Read Train Dataset and Divide Train/Val Dataset\n",
    "\"\"\"\n",
    "img_root_path = 'fashion-mnist/data/fashion'\n",
    "num_class = 10\n",
    "\n",
    "img, label = load_mnist(img_root_path, kind='train')\n",
    "\n",
    "for i in range(num_class):\n",
    "    class_img = img[np.where(label==i), :][0]\n",
    "    class_img = np.reshape(class_img, (class_img.shape[0], 28, 28))\n",
    "    \n",
    "    print('Class {} Number: {}'.format(i, class_img.shape[0]))\n",
    "\n",
    "    np.random.shuffle(class_img)\n",
    "\n",
    "    train_num = int(class_img.shape[0] * 0.8)\n",
    "\n",
    "    train_img = class_img[:train_num, : ,:]\n",
    "    train_label = np.zeros(train_num)\n",
    "    train_label[:] = i\n",
    "\n",
    "    val_img = class_img[train_num:, :, :]\n",
    "    val_label = np.zeros(val_img.shape[0])\n",
    "    val_label[:] = i\n",
    "\n",
    "    if i == 0:\n",
    "        train_img_total = train_img\n",
    "        train_label_total = train_label\n",
    "        val_img_total = val_img\n",
    "        val_label_total = val_label\n",
    "    else:\n",
    "        train_img_total = np.concatenate((train_img_total, train_img), axis=0)\n",
    "        train_label_total = np.concatenate((train_label_total, train_label), axis=0)\n",
    "        val_img_total = np.concatenate((val_img_total, val_img), axis=0)\n",
    "        val_label_total = np.concatenate((val_label_total, val_label), axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Read Test Dataset\n",
    "\"\"\"\n",
    "test_img, test_label = load_mnist(img_root_path, kind='t10k')\n",
    "test_img = np.reshape(test_img, (test_img.shape[0], 28, 28))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Define Image Dataset Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils import data\n",
    "from PIL import Image\n",
    "\n",
    "# define dataset class\n",
    "class Dataset(data.Dataset):\n",
    "    def __init__(self, img_array, label_array, transform=None):\n",
    "        self.transform = transform\n",
    "\n",
    "        self.img_array = img_array\n",
    "        self.label_array = label_array\n",
    "\n",
    "    # get number of data\n",
    "    def __len__(self):\n",
    "        return self.img_array.shape[0]\n",
    "\n",
    "    # IO with dataloader\n",
    "    def __getitem__(self, idx):\n",
    "        img = Image.fromarray(self.img_array[idx])\n",
    "        img = img.convert('L')\n",
    "\n",
    "        label = int(self.label_array[idx])\n",
    "\n",
    "        # transform data\n",
    "        if self.transform is not None:\n",
    "            img = self.transform(img)\n",
    "\n",
    "        return img, label"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Set Transform and Data Loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchvision import transforms\n",
    "import torch\n",
    "\n",
    "batch_size = 32\n",
    "\n",
    "# set image array\n",
    "img_array = {'train': train_img_total, 'val': val_img_total}\n",
    "\n",
    "# set label array\n",
    "label_array = {'train': train_label_total, 'val': val_label_total}\n",
    "\n",
    "# data augmentation\n",
    "data_transform = {\n",
    "    'train': transforms.Compose([\n",
    "        transforms.RandomHorizontalFlip(), \n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize((0.1307,), (0.3081,)),\n",
    "    ]),\n",
    "    'val': transforms.Compose([\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize((0.1307,), (0.3081,)),\n",
    "    ])\n",
    "}\n",
    "\n",
    "# image dataset\n",
    "image_dataset = {x: Dataset(img_array[x], label_array[x], data_transform[x]) for x in ['train', 'val']}\n",
    "\n",
    "# data loader\n",
    "data_loader = {x: torch.utils.data.DataLoader(image_dataset[x], batch_size=batch_size, shuffle=True, num_workers=4)\n",
    "                for x in ['train', 'val']}\n",
    "\n",
    "dataset_size = {x: len(image_dataset[x]) for x in ['train', 'val']}\n",
    "\n",
    "# check GPU\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Train Function\n",
    "\n",
    "Find Model Parameters for Getting Minimum Validation Loss\n",
    "\n",
    "Hyperparameters of Optimizer is Changed by Validation Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "\n",
    "def train(model, criterion, optimizer, scheduler, save_file_path):\n",
    "    # set epoch\n",
    "    num_epoch = 100\n",
    "\n",
    "    best_model_wt = copy.deepcopy(model.state_dict())\n",
    "    best_loss = np.inf\n",
    "\n",
    "    # count how many epoch validation loss is not improved\n",
    "    patience = 0\n",
    "\n",
    "    for epoch in range(num_epoch):\n",
    "        # check patience for stopping \n",
    "        if patience == 10:\n",
    "            break\n",
    "            \n",
    "        print('Epoch {}/{}'.format(epoch+1, num_epoch))\n",
    "        \n",
    "        # each epoch has train and validation phase\n",
    "        for phase in ['train', 'val']:\n",
    "            # train mode\n",
    "            if phase == 'train':\n",
    "                model.train()\n",
    "            # evaluation mode\n",
    "            else:\n",
    "                model.eval()\n",
    "                \n",
    "            epoch_loss = 0.0\n",
    "            epoch_correct = 0\n",
    "            \n",
    "            # iteration by batch size\n",
    "            for img, label in data_loader[phase]:\n",
    "                img = img.to(device)\n",
    "                label = label.to(device)\n",
    "                \n",
    "                # zero gradient parameter\n",
    "                optimizer.zero_grad()\n",
    "                \n",
    "                # forward network\n",
    "                with torch.set_grad_enabled(phase=='train'):\n",
    "                    output = model(img)\n",
    "                    _, pred = torch.max(output, 1)\n",
    "                    loss = criterion(output, label)\n",
    "                    \n",
    "                    # back propagation and optimize only train phase\n",
    "                    if phase == 'train':\n",
    "                        loss.backward()\n",
    "                        optimizer.step()\n",
    "                        \n",
    "                    epoch_loss += loss.item() * img.size(0)\n",
    "                    epoch_correct += torch.sum(pred == label.data)\n",
    "                    \n",
    "            epoch_loss /= dataset_size[phase]\n",
    "            epoch_acc = epoch_correct.double() / dataset_size[phase]\n",
    "            \n",
    "            # save weight by getting min validation loss\n",
    "            if phase == 'val':\n",
    "                scheduler.step(epoch_loss)\n",
    "                \n",
    "                if epoch_loss < best_loss:\n",
    "                    best_loss = epoch_loss\n",
    "                    best_model_wt = copy.deepcopy(model.state_dict())\n",
    "                    model.load_state_dict(best_model_wt)\n",
    "                    \n",
    "                    # save file path\n",
    "                    torch.save(model.state_dict(), save_file_path)\n",
    "                    \n",
    "                    # reset patience count\n",
    "                    patience = 0\n",
    "                else:\n",
    "                    patience += 1\n",
    "                    \n",
    "                print('Learning Rate: {}'.format(optimizer.param_groups[0]['lr']))\n",
    "                print('Best Validation Loss: {}'.format(best_loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Test Function\n",
    "\n",
    "Model Prediction and Evaluate\n",
    "\n",
    "Check Processing time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "def test(model, save_file_path):\n",
    "    # load best weight model\n",
    "    model.load_state_dict(torch.load(save_file_path))\n",
    "    model.eval()\n",
    "\n",
    "    # confusing matrix\n",
    "    cm = np.zeros((num_class, num_class), dtype = np.int32)\n",
    "\n",
    "    # transform for prediction\n",
    "    transform_tensor = transforms.ToTensor()\n",
    "    transform_normalize = transforms.Normalize((0.1307,), (0.3081,))\n",
    "\n",
    "    correct = 0\n",
    "\n",
    "    since = time.time()\n",
    "\n",
    "    for idx in range(test_img.shape[0]):\n",
    "        img = Image.fromarray(test_img[idx])\n",
    "        img = img.convert('L')\n",
    "        img = transform_tensor(img)\n",
    "        img = transform_normalize(img)\n",
    "\n",
    "        # add dim(batch) for matching train model\n",
    "        img = img.view(1, img.shape[0], img.shape[1], img.shape[2])\n",
    "        img = img.to(device)\n",
    "\n",
    "        label = int(test_label[idx])\n",
    "\n",
    "        # forward network\n",
    "        output = model(img)\n",
    "        _, pred = torch.max(output, 1)\n",
    "\n",
    "        # count precision, recall on confusion matrix\n",
    "        cm[label, int(pred)] += 1\n",
    "\n",
    "        correct += int(pred) == label\n",
    "\n",
    "    processing_time = time.time() - since\n",
    "    processing_time /= test_img.shape[0]\n",
    "\n",
    "    accuracy = float(correct) / test_img.shape[0]\n",
    "    \n",
    "    for i in range(num_class):\n",
    "        line = []\n",
    "        for j in range(num_class):\n",
    "            line.append('{:4d}'.format(cm[i, j]))\n",
    "            \n",
    "        print(' '.join(line))\n",
    "\n",
    "    print('Accuracy: {:.4f}'.format(accuracy))\n",
    "    print('Time per Image: {:.4f}'.format(processing_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6. Model Selection\n",
    "\n",
    "Find Effective Model using torchsummary package\n",
    "\n",
    "Candidate Model are Choosen Resnet18, Mobilenetv2, Condensenet\n",
    "\n",
    "Resnet18 and Mobilenetv2 from https://github.com/pytorch/vision/blob/master/torchvision/models\n",
    "\n",
    "Condensenet from https://github.com/ShichenLiu/CondenseNet\n",
    "\n",
    "Among Them, I would like to choose Mobilenetv2 and Condensenet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Conv2d-1           [-1, 64, 14, 14]           3,136\n",
      "       BatchNorm2d-2           [-1, 64, 14, 14]             128\n",
      "              ReLU-3           [-1, 64, 14, 14]               0\n",
      "         MaxPool2d-4             [-1, 64, 7, 7]               0\n",
      "            Conv2d-5             [-1, 64, 7, 7]          36,864\n",
      "       BatchNorm2d-6             [-1, 64, 7, 7]             128\n",
      "              ReLU-7             [-1, 64, 7, 7]               0\n",
      "            Conv2d-8             [-1, 64, 7, 7]          36,864\n",
      "       BatchNorm2d-9             [-1, 64, 7, 7]             128\n",
      "             ReLU-10             [-1, 64, 7, 7]               0\n",
      "       BasicBlock-11             [-1, 64, 7, 7]               0\n",
      "           Conv2d-12             [-1, 64, 7, 7]          36,864\n",
      "      BatchNorm2d-13             [-1, 64, 7, 7]             128\n",
      "             ReLU-14             [-1, 64, 7, 7]               0\n",
      "           Conv2d-15             [-1, 64, 7, 7]          36,864\n",
      "      BatchNorm2d-16             [-1, 64, 7, 7]             128\n",
      "             ReLU-17             [-1, 64, 7, 7]               0\n",
      "       BasicBlock-18             [-1, 64, 7, 7]               0\n",
      "           Conv2d-19            [-1, 128, 4, 4]          73,728\n",
      "      BatchNorm2d-20            [-1, 128, 4, 4]             256\n",
      "             ReLU-21            [-1, 128, 4, 4]               0\n",
      "           Conv2d-22            [-1, 128, 4, 4]         147,456\n",
      "      BatchNorm2d-23            [-1, 128, 4, 4]             256\n",
      "           Conv2d-24            [-1, 128, 4, 4]           8,192\n",
      "      BatchNorm2d-25            [-1, 128, 4, 4]             256\n",
      "             ReLU-26            [-1, 128, 4, 4]               0\n",
      "       BasicBlock-27            [-1, 128, 4, 4]               0\n",
      "           Conv2d-28            [-1, 128, 4, 4]         147,456\n",
      "      BatchNorm2d-29            [-1, 128, 4, 4]             256\n",
      "             ReLU-30            [-1, 128, 4, 4]               0\n",
      "           Conv2d-31            [-1, 128, 4, 4]         147,456\n",
      "      BatchNorm2d-32            [-1, 128, 4, 4]             256\n",
      "             ReLU-33            [-1, 128, 4, 4]               0\n",
      "       BasicBlock-34            [-1, 128, 4, 4]               0\n",
      "           Conv2d-35            [-1, 256, 2, 2]         294,912\n",
      "      BatchNorm2d-36            [-1, 256, 2, 2]             512\n",
      "             ReLU-37            [-1, 256, 2, 2]               0\n",
      "           Conv2d-38            [-1, 256, 2, 2]         589,824\n",
      "      BatchNorm2d-39            [-1, 256, 2, 2]             512\n",
      "           Conv2d-40            [-1, 256, 2, 2]          32,768\n",
      "      BatchNorm2d-41            [-1, 256, 2, 2]             512\n",
      "             ReLU-42            [-1, 256, 2, 2]               0\n",
      "       BasicBlock-43            [-1, 256, 2, 2]               0\n",
      "           Conv2d-44            [-1, 256, 2, 2]         589,824\n",
      "      BatchNorm2d-45            [-1, 256, 2, 2]             512\n",
      "             ReLU-46            [-1, 256, 2, 2]               0\n",
      "           Conv2d-47            [-1, 256, 2, 2]         589,824\n",
      "      BatchNorm2d-48            [-1, 256, 2, 2]             512\n",
      "             ReLU-49            [-1, 256, 2, 2]               0\n",
      "       BasicBlock-50            [-1, 256, 2, 2]               0\n",
      "           Conv2d-51            [-1, 512, 1, 1]       1,179,648\n",
      "      BatchNorm2d-52            [-1, 512, 1, 1]           1,024\n",
      "             ReLU-53            [-1, 512, 1, 1]               0\n",
      "           Conv2d-54            [-1, 512, 1, 1]       2,359,296\n",
      "      BatchNorm2d-55            [-1, 512, 1, 1]           1,024\n",
      "           Conv2d-56            [-1, 512, 1, 1]         131,072\n",
      "      BatchNorm2d-57            [-1, 512, 1, 1]           1,024\n",
      "             ReLU-58            [-1, 512, 1, 1]               0\n",
      "       BasicBlock-59            [-1, 512, 1, 1]               0\n",
      "           Conv2d-60            [-1, 512, 1, 1]       2,359,296\n",
      "      BatchNorm2d-61            [-1, 512, 1, 1]           1,024\n",
      "             ReLU-62            [-1, 512, 1, 1]               0\n",
      "           Conv2d-63            [-1, 512, 1, 1]       2,359,296\n",
      "      BatchNorm2d-64            [-1, 512, 1, 1]           1,024\n",
      "             ReLU-65            [-1, 512, 1, 1]               0\n",
      "       BasicBlock-66            [-1, 512, 1, 1]               0\n",
      "AdaptiveAvgPool2d-67            [-1, 512, 1, 1]               0\n",
      "           Linear-68                   [-1, 10]           5,130\n",
      "================================================================\n",
      "Total params: 11,175,370\n",
      "Trainable params: 11,175,370\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.00\n",
      "Forward/backward pass size (MB): 1.09\n",
      "Params size (MB): 42.63\n",
      "Estimated Total Size (MB): 43.72\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "from torchsummary import summary\n",
    "import torch\n",
    "import torchvision\n",
    "import torch.nn as nn\n",
    "\n",
    "\"\"\"\n",
    "Resnet18\n",
    "\"\"\"\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "model = torchvision.models.resnet18()\n",
    "\n",
    "# modified first layer for gray image\n",
    "model.conv1 = torch.nn.Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), \n",
    "            padding=(3, 3), bias=False)\n",
    "\n",
    "# modified last layer for changing class\n",
    "num_ftr = model.fc.in_features\n",
    "model.fc = nn.Linear(num_ftr, num_class)\n",
    "\n",
    "model = model.to(device)\n",
    "\n",
    "summary(model, (1, 28, 28))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Conv2d-1           [-1, 32, 14, 14]             288\n",
      "       BatchNorm2d-2           [-1, 32, 14, 14]              64\n",
      "             ReLU6-3           [-1, 32, 14, 14]               0\n",
      "            Conv2d-4           [-1, 32, 14, 14]             288\n",
      "       BatchNorm2d-5           [-1, 32, 14, 14]              64\n",
      "             ReLU6-6           [-1, 32, 14, 14]               0\n",
      "            Conv2d-7           [-1, 16, 14, 14]             512\n",
      "       BatchNorm2d-8           [-1, 16, 14, 14]              32\n",
      "  InvertedResidual-9           [-1, 16, 14, 14]               0\n",
      "           Conv2d-10           [-1, 96, 14, 14]           1,536\n",
      "      BatchNorm2d-11           [-1, 96, 14, 14]             192\n",
      "            ReLU6-12           [-1, 96, 14, 14]               0\n",
      "           Conv2d-13             [-1, 96, 7, 7]             864\n",
      "      BatchNorm2d-14             [-1, 96, 7, 7]             192\n",
      "            ReLU6-15             [-1, 96, 7, 7]               0\n",
      "           Conv2d-16             [-1, 24, 7, 7]           2,304\n",
      "      BatchNorm2d-17             [-1, 24, 7, 7]              48\n",
      " InvertedResidual-18             [-1, 24, 7, 7]               0\n",
      "           Conv2d-19            [-1, 144, 7, 7]           3,456\n",
      "      BatchNorm2d-20            [-1, 144, 7, 7]             288\n",
      "            ReLU6-21            [-1, 144, 7, 7]               0\n",
      "           Conv2d-22            [-1, 144, 7, 7]           1,296\n",
      "      BatchNorm2d-23            [-1, 144, 7, 7]             288\n",
      "            ReLU6-24            [-1, 144, 7, 7]               0\n",
      "           Conv2d-25             [-1, 24, 7, 7]           3,456\n",
      "      BatchNorm2d-26             [-1, 24, 7, 7]              48\n",
      " InvertedResidual-27             [-1, 24, 7, 7]               0\n",
      "           Conv2d-28            [-1, 144, 7, 7]           3,456\n",
      "      BatchNorm2d-29            [-1, 144, 7, 7]             288\n",
      "            ReLU6-30            [-1, 144, 7, 7]               0\n",
      "           Conv2d-31            [-1, 144, 4, 4]           1,296\n",
      "      BatchNorm2d-32            [-1, 144, 4, 4]             288\n",
      "            ReLU6-33            [-1, 144, 4, 4]               0\n",
      "           Conv2d-34             [-1, 32, 4, 4]           4,608\n",
      "      BatchNorm2d-35             [-1, 32, 4, 4]              64\n",
      " InvertedResidual-36             [-1, 32, 4, 4]               0\n",
      "           Conv2d-37            [-1, 192, 4, 4]           6,144\n",
      "      BatchNorm2d-38            [-1, 192, 4, 4]             384\n",
      "            ReLU6-39            [-1, 192, 4, 4]               0\n",
      "           Conv2d-40            [-1, 192, 4, 4]           1,728\n",
      "      BatchNorm2d-41            [-1, 192, 4, 4]             384\n",
      "            ReLU6-42            [-1, 192, 4, 4]               0\n",
      "           Conv2d-43             [-1, 32, 4, 4]           6,144\n",
      "      BatchNorm2d-44             [-1, 32, 4, 4]              64\n",
      " InvertedResidual-45             [-1, 32, 4, 4]               0\n",
      "           Conv2d-46            [-1, 192, 4, 4]           6,144\n",
      "      BatchNorm2d-47            [-1, 192, 4, 4]             384\n",
      "            ReLU6-48            [-1, 192, 4, 4]               0\n",
      "           Conv2d-49            [-1, 192, 4, 4]           1,728\n",
      "      BatchNorm2d-50            [-1, 192, 4, 4]             384\n",
      "            ReLU6-51            [-1, 192, 4, 4]               0\n",
      "           Conv2d-52             [-1, 32, 4, 4]           6,144\n",
      "      BatchNorm2d-53             [-1, 32, 4, 4]              64\n",
      " InvertedResidual-54             [-1, 32, 4, 4]               0\n",
      "           Conv2d-55            [-1, 192, 4, 4]           6,144\n",
      "      BatchNorm2d-56            [-1, 192, 4, 4]             384\n",
      "            ReLU6-57            [-1, 192, 4, 4]               0\n",
      "           Conv2d-58            [-1, 192, 2, 2]           1,728\n",
      "      BatchNorm2d-59            [-1, 192, 2, 2]             384\n",
      "            ReLU6-60            [-1, 192, 2, 2]               0\n",
      "           Conv2d-61             [-1, 64, 2, 2]          12,288\n",
      "      BatchNorm2d-62             [-1, 64, 2, 2]             128\n",
      " InvertedResidual-63             [-1, 64, 2, 2]               0\n",
      "           Conv2d-64            [-1, 384, 2, 2]          24,576\n",
      "      BatchNorm2d-65            [-1, 384, 2, 2]             768\n",
      "            ReLU6-66            [-1, 384, 2, 2]               0\n",
      "           Conv2d-67            [-1, 384, 2, 2]           3,456\n",
      "      BatchNorm2d-68            [-1, 384, 2, 2]             768\n",
      "            ReLU6-69            [-1, 384, 2, 2]               0\n",
      "           Conv2d-70             [-1, 64, 2, 2]          24,576\n",
      "      BatchNorm2d-71             [-1, 64, 2, 2]             128\n",
      " InvertedResidual-72             [-1, 64, 2, 2]               0\n",
      "           Conv2d-73            [-1, 384, 2, 2]          24,576\n",
      "      BatchNorm2d-74            [-1, 384, 2, 2]             768\n",
      "            ReLU6-75            [-1, 384, 2, 2]               0\n",
      "           Conv2d-76            [-1, 384, 2, 2]           3,456\n",
      "      BatchNorm2d-77            [-1, 384, 2, 2]             768\n",
      "            ReLU6-78            [-1, 384, 2, 2]               0\n",
      "           Conv2d-79             [-1, 64, 2, 2]          24,576\n",
      "      BatchNorm2d-80             [-1, 64, 2, 2]             128\n",
      " InvertedResidual-81             [-1, 64, 2, 2]               0\n",
      "           Conv2d-82            [-1, 384, 2, 2]          24,576\n",
      "      BatchNorm2d-83            [-1, 384, 2, 2]             768\n",
      "            ReLU6-84            [-1, 384, 2, 2]               0\n",
      "           Conv2d-85            [-1, 384, 2, 2]           3,456\n",
      "      BatchNorm2d-86            [-1, 384, 2, 2]             768\n",
      "            ReLU6-87            [-1, 384, 2, 2]               0\n",
      "           Conv2d-88             [-1, 64, 2, 2]          24,576\n",
      "      BatchNorm2d-89             [-1, 64, 2, 2]             128\n",
      " InvertedResidual-90             [-1, 64, 2, 2]               0\n",
      "           Conv2d-91            [-1, 384, 2, 2]          24,576\n",
      "      BatchNorm2d-92            [-1, 384, 2, 2]             768\n",
      "            ReLU6-93            [-1, 384, 2, 2]               0\n",
      "           Conv2d-94            [-1, 384, 2, 2]           3,456\n",
      "      BatchNorm2d-95            [-1, 384, 2, 2]             768\n",
      "            ReLU6-96            [-1, 384, 2, 2]               0\n",
      "           Conv2d-97             [-1, 96, 2, 2]          36,864\n",
      "      BatchNorm2d-98             [-1, 96, 2, 2]             192\n",
      " InvertedResidual-99             [-1, 96, 2, 2]               0\n",
      "          Conv2d-100            [-1, 576, 2, 2]          55,296\n",
      "     BatchNorm2d-101            [-1, 576, 2, 2]           1,152\n",
      "           ReLU6-102            [-1, 576, 2, 2]               0\n",
      "          Conv2d-103            [-1, 576, 2, 2]           5,184\n",
      "     BatchNorm2d-104            [-1, 576, 2, 2]           1,152\n",
      "           ReLU6-105            [-1, 576, 2, 2]               0\n",
      "          Conv2d-106             [-1, 96, 2, 2]          55,296\n",
      "     BatchNorm2d-107             [-1, 96, 2, 2]             192\n",
      "InvertedResidual-108             [-1, 96, 2, 2]               0\n",
      "          Conv2d-109            [-1, 576, 2, 2]          55,296\n",
      "     BatchNorm2d-110            [-1, 576, 2, 2]           1,152\n",
      "           ReLU6-111            [-1, 576, 2, 2]               0\n",
      "          Conv2d-112            [-1, 576, 2, 2]           5,184\n",
      "     BatchNorm2d-113            [-1, 576, 2, 2]           1,152\n",
      "           ReLU6-114            [-1, 576, 2, 2]               0\n",
      "          Conv2d-115             [-1, 96, 2, 2]          55,296\n",
      "     BatchNorm2d-116             [-1, 96, 2, 2]             192\n",
      "InvertedResidual-117             [-1, 96, 2, 2]               0\n",
      "          Conv2d-118            [-1, 576, 2, 2]          55,296\n",
      "     BatchNorm2d-119            [-1, 576, 2, 2]           1,152\n",
      "           ReLU6-120            [-1, 576, 2, 2]               0\n",
      "          Conv2d-121            [-1, 576, 1, 1]           5,184\n",
      "     BatchNorm2d-122            [-1, 576, 1, 1]           1,152\n",
      "           ReLU6-123            [-1, 576, 1, 1]               0\n",
      "          Conv2d-124            [-1, 160, 1, 1]          92,160\n",
      "     BatchNorm2d-125            [-1, 160, 1, 1]             320\n",
      "InvertedResidual-126            [-1, 160, 1, 1]               0\n",
      "          Conv2d-127            [-1, 960, 1, 1]         153,600\n",
      "     BatchNorm2d-128            [-1, 960, 1, 1]           1,920\n",
      "           ReLU6-129            [-1, 960, 1, 1]               0\n",
      "          Conv2d-130            [-1, 960, 1, 1]           8,640\n",
      "     BatchNorm2d-131            [-1, 960, 1, 1]           1,920\n",
      "           ReLU6-132            [-1, 960, 1, 1]               0\n",
      "          Conv2d-133            [-1, 160, 1, 1]         153,600\n",
      "     BatchNorm2d-134            [-1, 160, 1, 1]             320\n",
      "InvertedResidual-135            [-1, 160, 1, 1]               0\n",
      "          Conv2d-136            [-1, 960, 1, 1]         153,600\n",
      "     BatchNorm2d-137            [-1, 960, 1, 1]           1,920\n",
      "           ReLU6-138            [-1, 960, 1, 1]               0\n",
      "          Conv2d-139            [-1, 960, 1, 1]           8,640\n",
      "     BatchNorm2d-140            [-1, 960, 1, 1]           1,920\n",
      "           ReLU6-141            [-1, 960, 1, 1]               0\n",
      "          Conv2d-142            [-1, 160, 1, 1]         153,600\n",
      "     BatchNorm2d-143            [-1, 160, 1, 1]             320\n",
      "InvertedResidual-144            [-1, 160, 1, 1]               0\n",
      "          Conv2d-145            [-1, 960, 1, 1]         153,600\n",
      "     BatchNorm2d-146            [-1, 960, 1, 1]           1,920\n",
      "           ReLU6-147            [-1, 960, 1, 1]               0\n",
      "          Conv2d-148            [-1, 960, 1, 1]           8,640\n",
      "     BatchNorm2d-149            [-1, 960, 1, 1]           1,920\n",
      "           ReLU6-150            [-1, 960, 1, 1]               0\n",
      "          Conv2d-151            [-1, 320, 1, 1]         307,200\n",
      "     BatchNorm2d-152            [-1, 320, 1, 1]             640\n",
      "InvertedResidual-153            [-1, 320, 1, 1]               0\n",
      "          Conv2d-154           [-1, 1280, 1, 1]         409,600\n",
      "     BatchNorm2d-155           [-1, 1280, 1, 1]           2,560\n",
      "           ReLU6-156           [-1, 1280, 1, 1]               0\n",
      "         Dropout-157                 [-1, 1280]               0\n",
      "          Linear-158                   [-1, 10]          12,810\n",
      "================================================================\n",
      "Total params: 2,236,106\n",
      "Trainable params: 2,236,106\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.00\n",
      "Forward/backward pass size (MB): 2.69\n",
      "Params size (MB): 8.53\n",
      "Estimated Total Size (MB): 11.22\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "MobilenetV2\n",
    "\"\"\"\n",
    "# already modified model for gray image\n",
    "from mobilenetv2 import mobilenet_v2\n",
    "\n",
    "model = mobilenet_v2()\n",
    "model = model.to(device)\n",
    "\n",
    "summary(model, (1, 28, 28))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Conv2d-1           [-1, 16, 28, 28]             144\n",
      "       BatchNorm2d-2           [-1, 16, 28, 28]              32\n",
      "              ReLU-3           [-1, 16, 28, 28]               0\n",
      "  LearnedGroupConv-4           [-1, 32, 28, 28]               0\n",
      "       BatchNorm2d-5           [-1, 32, 28, 28]              64\n",
      "              ReLU-6           [-1, 32, 28, 28]               0\n",
      "            Conv2d-7            [-1, 8, 28, 28]             576\n",
      "       _DenseLayer-8           [-1, 24, 28, 28]               0\n",
      "       BatchNorm2d-9           [-1, 24, 28, 28]              48\n",
      "             ReLU-10           [-1, 24, 28, 28]               0\n",
      " LearnedGroupConv-11           [-1, 32, 28, 28]               0\n",
      "      BatchNorm2d-12           [-1, 32, 28, 28]              64\n",
      "             ReLU-13           [-1, 32, 28, 28]               0\n",
      "           Conv2d-14            [-1, 8, 28, 28]             576\n",
      "      _DenseLayer-15           [-1, 32, 28, 28]               0\n",
      "      BatchNorm2d-16           [-1, 32, 28, 28]              64\n",
      "             ReLU-17           [-1, 32, 28, 28]               0\n",
      " LearnedGroupConv-18           [-1, 32, 28, 28]               0\n",
      "      BatchNorm2d-19           [-1, 32, 28, 28]              64\n",
      "             ReLU-20           [-1, 32, 28, 28]               0\n",
      "           Conv2d-21            [-1, 8, 28, 28]             576\n",
      "      _DenseLayer-22           [-1, 40, 28, 28]               0\n",
      "      BatchNorm2d-23           [-1, 40, 28, 28]              80\n",
      "             ReLU-24           [-1, 40, 28, 28]               0\n",
      " LearnedGroupConv-25           [-1, 32, 28, 28]               0\n",
      "      BatchNorm2d-26           [-1, 32, 28, 28]              64\n",
      "             ReLU-27           [-1, 32, 28, 28]               0\n",
      "           Conv2d-28            [-1, 8, 28, 28]             576\n",
      "      _DenseLayer-29           [-1, 48, 28, 28]               0\n",
      "      BatchNorm2d-30           [-1, 48, 28, 28]              96\n",
      "             ReLU-31           [-1, 48, 28, 28]               0\n",
      " LearnedGroupConv-32           [-1, 32, 28, 28]               0\n",
      "      BatchNorm2d-33           [-1, 32, 28, 28]              64\n",
      "             ReLU-34           [-1, 32, 28, 28]               0\n",
      "           Conv2d-35            [-1, 8, 28, 28]             576\n",
      "      _DenseLayer-36           [-1, 56, 28, 28]               0\n",
      "      BatchNorm2d-37           [-1, 56, 28, 28]             112\n",
      "             ReLU-38           [-1, 56, 28, 28]               0\n",
      " LearnedGroupConv-39           [-1, 32, 28, 28]               0\n",
      "      BatchNorm2d-40           [-1, 32, 28, 28]              64\n",
      "             ReLU-41           [-1, 32, 28, 28]               0\n",
      "           Conv2d-42            [-1, 8, 28, 28]             576\n",
      "      _DenseLayer-43           [-1, 64, 28, 28]               0\n",
      "      BatchNorm2d-44           [-1, 64, 28, 28]             128\n",
      "             ReLU-45           [-1, 64, 28, 28]               0\n",
      " LearnedGroupConv-46           [-1, 32, 28, 28]               0\n",
      "      BatchNorm2d-47           [-1, 32, 28, 28]              64\n",
      "             ReLU-48           [-1, 32, 28, 28]               0\n",
      "           Conv2d-49            [-1, 8, 28, 28]             576\n",
      "      _DenseLayer-50           [-1, 72, 28, 28]               0\n",
      "      BatchNorm2d-51           [-1, 72, 28, 28]             144\n",
      "             ReLU-52           [-1, 72, 28, 28]               0\n",
      " LearnedGroupConv-53           [-1, 32, 28, 28]               0\n",
      "      BatchNorm2d-54           [-1, 32, 28, 28]              64\n",
      "             ReLU-55           [-1, 32, 28, 28]               0\n",
      "           Conv2d-56            [-1, 8, 28, 28]             576\n",
      "      _DenseLayer-57           [-1, 80, 28, 28]               0\n",
      "      BatchNorm2d-58           [-1, 80, 28, 28]             160\n",
      "             ReLU-59           [-1, 80, 28, 28]               0\n",
      " LearnedGroupConv-60           [-1, 32, 28, 28]               0\n",
      "      BatchNorm2d-61           [-1, 32, 28, 28]              64\n",
      "             ReLU-62           [-1, 32, 28, 28]               0\n",
      "           Conv2d-63            [-1, 8, 28, 28]             576\n",
      "      _DenseLayer-64           [-1, 88, 28, 28]               0\n",
      "      BatchNorm2d-65           [-1, 88, 28, 28]             176\n",
      "             ReLU-66           [-1, 88, 28, 28]               0\n",
      " LearnedGroupConv-67           [-1, 32, 28, 28]               0\n",
      "      BatchNorm2d-68           [-1, 32, 28, 28]              64\n",
      "             ReLU-69           [-1, 32, 28, 28]               0\n",
      "           Conv2d-70            [-1, 8, 28, 28]             576\n",
      "      _DenseLayer-71           [-1, 96, 28, 28]               0\n",
      "      BatchNorm2d-72           [-1, 96, 28, 28]             192\n",
      "             ReLU-73           [-1, 96, 28, 28]               0\n",
      " LearnedGroupConv-74           [-1, 32, 28, 28]               0\n",
      "      BatchNorm2d-75           [-1, 32, 28, 28]              64\n",
      "             ReLU-76           [-1, 32, 28, 28]               0\n",
      "           Conv2d-77            [-1, 8, 28, 28]             576\n",
      "      _DenseLayer-78          [-1, 104, 28, 28]               0\n",
      "      BatchNorm2d-79          [-1, 104, 28, 28]             208\n",
      "             ReLU-80          [-1, 104, 28, 28]               0\n",
      " LearnedGroupConv-81           [-1, 32, 28, 28]               0\n",
      "      BatchNorm2d-82           [-1, 32, 28, 28]              64\n",
      "             ReLU-83           [-1, 32, 28, 28]               0\n",
      "           Conv2d-84            [-1, 8, 28, 28]             576\n",
      "      _DenseLayer-85          [-1, 112, 28, 28]               0\n",
      "      BatchNorm2d-86          [-1, 112, 28, 28]             224\n",
      "             ReLU-87          [-1, 112, 28, 28]               0\n",
      " LearnedGroupConv-88           [-1, 32, 28, 28]               0\n",
      "      BatchNorm2d-89           [-1, 32, 28, 28]              64\n",
      "             ReLU-90           [-1, 32, 28, 28]               0\n",
      "           Conv2d-91            [-1, 8, 28, 28]             576\n",
      "      _DenseLayer-92          [-1, 120, 28, 28]               0\n",
      "      BatchNorm2d-93          [-1, 120, 28, 28]             240\n",
      "             ReLU-94          [-1, 120, 28, 28]               0\n",
      " LearnedGroupConv-95           [-1, 32, 28, 28]               0\n",
      "      BatchNorm2d-96           [-1, 32, 28, 28]              64\n",
      "             ReLU-97           [-1, 32, 28, 28]               0\n",
      "           Conv2d-98            [-1, 8, 28, 28]             576\n",
      "      _DenseLayer-99          [-1, 128, 28, 28]               0\n",
      "       AvgPool2d-100          [-1, 128, 14, 14]               0\n",
      "     _Transition-101          [-1, 128, 14, 14]               0\n",
      "     BatchNorm2d-102          [-1, 128, 14, 14]             256\n",
      "            ReLU-103          [-1, 128, 14, 14]               0\n",
      "LearnedGroupConv-104           [-1, 64, 14, 14]               0\n",
      "     BatchNorm2d-105           [-1, 64, 14, 14]             128\n",
      "            ReLU-106           [-1, 64, 14, 14]               0\n",
      "          Conv2d-107           [-1, 16, 14, 14]           2,304\n",
      "     _DenseLayer-108          [-1, 144, 14, 14]               0\n",
      "     BatchNorm2d-109          [-1, 144, 14, 14]             288\n",
      "            ReLU-110          [-1, 144, 14, 14]               0\n",
      "LearnedGroupConv-111           [-1, 64, 14, 14]               0\n",
      "     BatchNorm2d-112           [-1, 64, 14, 14]             128\n",
      "            ReLU-113           [-1, 64, 14, 14]               0\n",
      "          Conv2d-114           [-1, 16, 14, 14]           2,304\n",
      "     _DenseLayer-115          [-1, 160, 14, 14]               0\n",
      "     BatchNorm2d-116          [-1, 160, 14, 14]             320\n",
      "            ReLU-117          [-1, 160, 14, 14]               0\n",
      "LearnedGroupConv-118           [-1, 64, 14, 14]               0\n",
      "     BatchNorm2d-119           [-1, 64, 14, 14]             128\n",
      "            ReLU-120           [-1, 64, 14, 14]               0\n",
      "          Conv2d-121           [-1, 16, 14, 14]           2,304\n",
      "     _DenseLayer-122          [-1, 176, 14, 14]               0\n",
      "     BatchNorm2d-123          [-1, 176, 14, 14]             352\n",
      "            ReLU-124          [-1, 176, 14, 14]               0\n",
      "LearnedGroupConv-125           [-1, 64, 14, 14]               0\n",
      "     BatchNorm2d-126           [-1, 64, 14, 14]             128\n",
      "            ReLU-127           [-1, 64, 14, 14]               0\n",
      "          Conv2d-128           [-1, 16, 14, 14]           2,304\n",
      "     _DenseLayer-129          [-1, 192, 14, 14]               0\n",
      "     BatchNorm2d-130          [-1, 192, 14, 14]             384\n",
      "            ReLU-131          [-1, 192, 14, 14]               0\n",
      "LearnedGroupConv-132           [-1, 64, 14, 14]               0\n",
      "     BatchNorm2d-133           [-1, 64, 14, 14]             128\n",
      "            ReLU-134           [-1, 64, 14, 14]               0\n",
      "          Conv2d-135           [-1, 16, 14, 14]           2,304\n",
      "     _DenseLayer-136          [-1, 208, 14, 14]               0\n",
      "     BatchNorm2d-137          [-1, 208, 14, 14]             416\n",
      "            ReLU-138          [-1, 208, 14, 14]               0\n",
      "LearnedGroupConv-139           [-1, 64, 14, 14]               0\n",
      "     BatchNorm2d-140           [-1, 64, 14, 14]             128\n",
      "            ReLU-141           [-1, 64, 14, 14]               0\n",
      "          Conv2d-142           [-1, 16, 14, 14]           2,304\n",
      "     _DenseLayer-143          [-1, 224, 14, 14]               0\n",
      "     BatchNorm2d-144          [-1, 224, 14, 14]             448\n",
      "            ReLU-145          [-1, 224, 14, 14]               0\n",
      "LearnedGroupConv-146           [-1, 64, 14, 14]               0\n",
      "     BatchNorm2d-147           [-1, 64, 14, 14]             128\n",
      "            ReLU-148           [-1, 64, 14, 14]               0\n",
      "          Conv2d-149           [-1, 16, 14, 14]           2,304\n",
      "     _DenseLayer-150          [-1, 240, 14, 14]               0\n",
      "     BatchNorm2d-151          [-1, 240, 14, 14]             480\n",
      "            ReLU-152          [-1, 240, 14, 14]               0\n",
      "LearnedGroupConv-153           [-1, 64, 14, 14]               0\n",
      "     BatchNorm2d-154           [-1, 64, 14, 14]             128\n",
      "            ReLU-155           [-1, 64, 14, 14]               0\n",
      "          Conv2d-156           [-1, 16, 14, 14]           2,304\n",
      "     _DenseLayer-157          [-1, 256, 14, 14]               0\n",
      "     BatchNorm2d-158          [-1, 256, 14, 14]             512\n",
      "            ReLU-159          [-1, 256, 14, 14]               0\n",
      "LearnedGroupConv-160           [-1, 64, 14, 14]               0\n",
      "     BatchNorm2d-161           [-1, 64, 14, 14]             128\n",
      "            ReLU-162           [-1, 64, 14, 14]               0\n",
      "          Conv2d-163           [-1, 16, 14, 14]           2,304\n",
      "     _DenseLayer-164          [-1, 272, 14, 14]               0\n",
      "     BatchNorm2d-165          [-1, 272, 14, 14]             544\n",
      "            ReLU-166          [-1, 272, 14, 14]               0\n",
      "LearnedGroupConv-167           [-1, 64, 14, 14]               0\n",
      "     BatchNorm2d-168           [-1, 64, 14, 14]             128\n",
      "            ReLU-169           [-1, 64, 14, 14]               0\n",
      "          Conv2d-170           [-1, 16, 14, 14]           2,304\n",
      "     _DenseLayer-171          [-1, 288, 14, 14]               0\n",
      "     BatchNorm2d-172          [-1, 288, 14, 14]             576\n",
      "            ReLU-173          [-1, 288, 14, 14]               0\n",
      "LearnedGroupConv-174           [-1, 64, 14, 14]               0\n",
      "     BatchNorm2d-175           [-1, 64, 14, 14]             128\n",
      "            ReLU-176           [-1, 64, 14, 14]               0\n",
      "          Conv2d-177           [-1, 16, 14, 14]           2,304\n",
      "     _DenseLayer-178          [-1, 304, 14, 14]               0\n",
      "     BatchNorm2d-179          [-1, 304, 14, 14]             608\n",
      "            ReLU-180          [-1, 304, 14, 14]               0\n",
      "LearnedGroupConv-181           [-1, 64, 14, 14]               0\n",
      "     BatchNorm2d-182           [-1, 64, 14, 14]             128\n",
      "            ReLU-183           [-1, 64, 14, 14]               0\n",
      "          Conv2d-184           [-1, 16, 14, 14]           2,304\n",
      "     _DenseLayer-185          [-1, 320, 14, 14]               0\n",
      "     BatchNorm2d-186          [-1, 320, 14, 14]             640\n",
      "            ReLU-187          [-1, 320, 14, 14]               0\n",
      "LearnedGroupConv-188           [-1, 64, 14, 14]               0\n",
      "     BatchNorm2d-189           [-1, 64, 14, 14]             128\n",
      "            ReLU-190           [-1, 64, 14, 14]               0\n",
      "          Conv2d-191           [-1, 16, 14, 14]           2,304\n",
      "     _DenseLayer-192          [-1, 336, 14, 14]               0\n",
      "     BatchNorm2d-193          [-1, 336, 14, 14]             672\n",
      "            ReLU-194          [-1, 336, 14, 14]               0\n",
      "LearnedGroupConv-195           [-1, 64, 14, 14]               0\n",
      "     BatchNorm2d-196           [-1, 64, 14, 14]             128\n",
      "            ReLU-197           [-1, 64, 14, 14]               0\n",
      "          Conv2d-198           [-1, 16, 14, 14]           2,304\n",
      "     _DenseLayer-199          [-1, 352, 14, 14]               0\n",
      "       AvgPool2d-200            [-1, 352, 7, 7]               0\n",
      "     _Transition-201            [-1, 352, 7, 7]               0\n",
      "     BatchNorm2d-202            [-1, 352, 7, 7]             704\n",
      "            ReLU-203            [-1, 352, 7, 7]               0\n",
      "LearnedGroupConv-204            [-1, 128, 7, 7]               0\n",
      "     BatchNorm2d-205            [-1, 128, 7, 7]             256\n",
      "            ReLU-206            [-1, 128, 7, 7]               0\n",
      "          Conv2d-207             [-1, 32, 7, 7]           9,216\n",
      "     _DenseLayer-208            [-1, 384, 7, 7]               0\n",
      "     BatchNorm2d-209            [-1, 384, 7, 7]             768\n",
      "            ReLU-210            [-1, 384, 7, 7]               0\n",
      "LearnedGroupConv-211            [-1, 128, 7, 7]               0\n",
      "     BatchNorm2d-212            [-1, 128, 7, 7]             256\n",
      "            ReLU-213            [-1, 128, 7, 7]               0\n",
      "          Conv2d-214             [-1, 32, 7, 7]           9,216\n",
      "     _DenseLayer-215            [-1, 416, 7, 7]               0\n",
      "     BatchNorm2d-216            [-1, 416, 7, 7]             832\n",
      "            ReLU-217            [-1, 416, 7, 7]               0\n",
      "LearnedGroupConv-218            [-1, 128, 7, 7]               0\n",
      "     BatchNorm2d-219            [-1, 128, 7, 7]             256\n",
      "            ReLU-220            [-1, 128, 7, 7]               0\n",
      "          Conv2d-221             [-1, 32, 7, 7]           9,216\n",
      "     _DenseLayer-222            [-1, 448, 7, 7]               0\n",
      "     BatchNorm2d-223            [-1, 448, 7, 7]             896\n",
      "            ReLU-224            [-1, 448, 7, 7]               0\n",
      "LearnedGroupConv-225            [-1, 128, 7, 7]               0\n",
      "     BatchNorm2d-226            [-1, 128, 7, 7]             256\n",
      "            ReLU-227            [-1, 128, 7, 7]               0\n",
      "          Conv2d-228             [-1, 32, 7, 7]           9,216\n",
      "     _DenseLayer-229            [-1, 480, 7, 7]               0\n",
      "     BatchNorm2d-230            [-1, 480, 7, 7]             960\n",
      "            ReLU-231            [-1, 480, 7, 7]               0\n",
      "LearnedGroupConv-232            [-1, 128, 7, 7]               0\n",
      "     BatchNorm2d-233            [-1, 128, 7, 7]             256\n",
      "            ReLU-234            [-1, 128, 7, 7]               0\n",
      "          Conv2d-235             [-1, 32, 7, 7]           9,216\n",
      "     _DenseLayer-236            [-1, 512, 7, 7]               0\n",
      "     BatchNorm2d-237            [-1, 512, 7, 7]           1,024\n",
      "            ReLU-238            [-1, 512, 7, 7]               0\n",
      "LearnedGroupConv-239            [-1, 128, 7, 7]               0\n",
      "     BatchNorm2d-240            [-1, 128, 7, 7]             256\n",
      "            ReLU-241            [-1, 128, 7, 7]               0\n",
      "          Conv2d-242             [-1, 32, 7, 7]           9,216\n",
      "     _DenseLayer-243            [-1, 544, 7, 7]               0\n",
      "     BatchNorm2d-244            [-1, 544, 7, 7]           1,088\n",
      "            ReLU-245            [-1, 544, 7, 7]               0\n",
      "LearnedGroupConv-246            [-1, 128, 7, 7]               0\n",
      "     BatchNorm2d-247            [-1, 128, 7, 7]             256\n",
      "            ReLU-248            [-1, 128, 7, 7]               0\n",
      "          Conv2d-249             [-1, 32, 7, 7]           9,216\n",
      "     _DenseLayer-250            [-1, 576, 7, 7]               0\n",
      "     BatchNorm2d-251            [-1, 576, 7, 7]           1,152\n",
      "            ReLU-252            [-1, 576, 7, 7]               0\n",
      "LearnedGroupConv-253            [-1, 128, 7, 7]               0\n",
      "     BatchNorm2d-254            [-1, 128, 7, 7]             256\n",
      "            ReLU-255            [-1, 128, 7, 7]               0\n",
      "          Conv2d-256             [-1, 32, 7, 7]           9,216\n",
      "     _DenseLayer-257            [-1, 608, 7, 7]               0\n",
      "     BatchNorm2d-258            [-1, 608, 7, 7]           1,216\n",
      "            ReLU-259            [-1, 608, 7, 7]               0\n",
      "LearnedGroupConv-260            [-1, 128, 7, 7]               0\n",
      "     BatchNorm2d-261            [-1, 128, 7, 7]             256\n",
      "            ReLU-262            [-1, 128, 7, 7]               0\n",
      "          Conv2d-263             [-1, 32, 7, 7]           9,216\n",
      "     _DenseLayer-264            [-1, 640, 7, 7]               0\n",
      "     BatchNorm2d-265            [-1, 640, 7, 7]           1,280\n",
      "            ReLU-266            [-1, 640, 7, 7]               0\n",
      "LearnedGroupConv-267            [-1, 128, 7, 7]               0\n",
      "     BatchNorm2d-268            [-1, 128, 7, 7]             256\n",
      "            ReLU-269            [-1, 128, 7, 7]               0\n",
      "          Conv2d-270             [-1, 32, 7, 7]           9,216\n",
      "     _DenseLayer-271            [-1, 672, 7, 7]               0\n",
      "     BatchNorm2d-272            [-1, 672, 7, 7]           1,344\n",
      "            ReLU-273            [-1, 672, 7, 7]               0\n",
      "LearnedGroupConv-274            [-1, 128, 7, 7]               0\n",
      "     BatchNorm2d-275            [-1, 128, 7, 7]             256\n",
      "            ReLU-276            [-1, 128, 7, 7]               0\n",
      "          Conv2d-277             [-1, 32, 7, 7]           9,216\n",
      "     _DenseLayer-278            [-1, 704, 7, 7]               0\n",
      "     BatchNorm2d-279            [-1, 704, 7, 7]           1,408\n",
      "            ReLU-280            [-1, 704, 7, 7]               0\n",
      "LearnedGroupConv-281            [-1, 128, 7, 7]               0\n",
      "     BatchNorm2d-282            [-1, 128, 7, 7]             256\n",
      "            ReLU-283            [-1, 128, 7, 7]               0\n",
      "          Conv2d-284             [-1, 32, 7, 7]           9,216\n",
      "     _DenseLayer-285            [-1, 736, 7, 7]               0\n",
      "     BatchNorm2d-286            [-1, 736, 7, 7]           1,472\n",
      "            ReLU-287            [-1, 736, 7, 7]               0\n",
      "LearnedGroupConv-288            [-1, 128, 7, 7]               0\n",
      "     BatchNorm2d-289            [-1, 128, 7, 7]             256\n",
      "            ReLU-290            [-1, 128, 7, 7]               0\n",
      "          Conv2d-291             [-1, 32, 7, 7]           9,216\n",
      "     _DenseLayer-292            [-1, 768, 7, 7]               0\n",
      "     BatchNorm2d-293            [-1, 768, 7, 7]           1,536\n",
      "            ReLU-294            [-1, 768, 7, 7]               0\n",
      "LearnedGroupConv-295            [-1, 128, 7, 7]               0\n",
      "     BatchNorm2d-296            [-1, 128, 7, 7]             256\n",
      "            ReLU-297            [-1, 128, 7, 7]               0\n",
      "          Conv2d-298             [-1, 32, 7, 7]           9,216\n",
      "     _DenseLayer-299            [-1, 800, 7, 7]               0\n",
      "     BatchNorm2d-300            [-1, 800, 7, 7]           1,600\n",
      "            ReLU-301            [-1, 800, 7, 7]               0\n",
      "       AvgPool2d-302            [-1, 800, 1, 1]               0\n",
      "          Linear-303                   [-1, 10]           8,010\n",
      "================================================================\n",
      "Total params: 209,450\n",
      "Trainable params: 209,450\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.00\n",
      "Forward/backward pass size (MB): 58.21\n",
      "Params size (MB): 0.80\n",
      "Estimated Total Size (MB): 59.01\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Condensenet\n",
    "\"\"\"\n",
    "# already modified model for gray image\n",
    "from condensenet import condensenet\n",
    "\n",
    "model = condensenet()\n",
    "model = model.to(device)\n",
    "\n",
    "summary(model, (1, 28, 28))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7. Set Loss Function and Optimizer\n",
    "\n",
    "Loss Function : Cross Entropy for Classification\n",
    "\n",
    "Optimizer : SGD\n",
    "\n",
    "Scheduler : Reduce Learning Rate by Validation Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "from torch.optim import lr_scheduler\n",
    "\n",
    "# cross entrophy loss function\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "# SGD optimizer\n",
    "optimizer = optim.SGD(model.parameters(), lr = 0.01, momentum = 0.9)\n",
    "\n",
    "# schedule for changing learning rate by validation loss\n",
    "scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 8. Train and Test Mobilenet v2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.43746373959382373\n",
      "Epoch 2/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.4163805858095487\n",
      "Epoch 3/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.36118652508656185\n",
      "Epoch 4/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.33984858204921087\n",
      "Epoch 5/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.305906698624293\n",
      "Epoch 6/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.305906698624293\n",
      "Epoch 7/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.2874024902184804\n",
      "Epoch 8/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.2874024902184804\n",
      "Epoch 9/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.2696236774722735\n",
      "Epoch 10/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.2696236774722735\n",
      "Epoch 11/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.2696236774722735\n",
      "Epoch 12/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.2696236774722735\n",
      "Epoch 13/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.25607051078478493\n",
      "Epoch 14/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.2522235461870829\n",
      "Epoch 15/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.2522235461870829\n",
      "Epoch 16/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.2522235461870829\n",
      "Epoch 17/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.2522235461870829\n",
      "Epoch 18/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.2522235461870829\n",
      "Epoch 19/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.2522235461870829\n",
      "Epoch 20/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.24171262721220652\n",
      "Epoch 21/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.2402724553346634\n",
      "Epoch 22/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.2402724553346634\n",
      "Epoch 23/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.2402724553346634\n",
      "Epoch 24/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.2402724553346634\n",
      "Epoch 25/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.23864462651809057\n",
      "Epoch 26/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.23864462651809057\n",
      "Epoch 27/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.23864462651809057\n",
      "Epoch 28/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.23864462651809057\n",
      "Epoch 29/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.23864462651809057\n",
      "Epoch 30/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.23864462651809057\n",
      "Epoch 31/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.23864462651809057\n",
      "Epoch 32/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.23864462651809057\n",
      "Epoch 33/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.23864462651809057\n",
      "Epoch 34/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.23864462651809057\n",
      "Epoch 35/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.23864462651809057\n"
     ]
    }
   ],
   "source": [
    "model = mobilenet_v2()\n",
    "model = model.to(device)\n",
    "\n",
    "optimizer = optim.SGD(model.parameters(), lr = 0.01, momentum = 0.9)\n",
    "\n",
    "save_file_path = 'mobilenetv2.pth'\n",
    "\n",
    "train(model, criterion, optimizer, scheduler, save_file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 870    0   15    8    2    0  100    0    5    0\n",
      "   2  982    0   11    0    0    3    0    2    0\n",
      "  18    1  881    9   33    0   58    0    0    0\n",
      "  25    7   11  897   40    1   18    0    1    0\n",
      "   1    0   87   21  834    0   57    0    0    0\n",
      "   1    0    0    0    0  963    0   27    3    6\n",
      "  99    1   62   18   61    1  749    0    9    0\n",
      "   0    0    0    0    0    6    0  974    0   20\n",
      "   5    0    0    4    1    1    6    1  982    0\n",
      "   0    0    0    0    0    5    0   31    2  962\n",
      "Accuracy: 0.9094\n",
      "Time per Image: 0.0054\n"
     ]
    }
   ],
   "source": [
    "test(model, save_file_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9. Train and Test Condensenet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.42809804852803546\n",
      "Epoch 2/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.3083564596970876\n",
      "Epoch 3/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.292668576002121\n",
      "Epoch 4/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.2914826430678368\n",
      "Epoch 5/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.2544105856815974\n",
      "Epoch 6/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.24198614865541457\n",
      "Epoch 7/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.24198614865541457\n",
      "Epoch 8/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.24198614865541457\n",
      "Epoch 9/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.21633389941851297\n",
      "Epoch 10/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.20506977492570877\n",
      "Epoch 11/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.20506977492570877\n",
      "Epoch 12/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.20506977492570877\n",
      "Epoch 13/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.20506977492570877\n",
      "Epoch 14/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.20506977492570877\n",
      "Epoch 15/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.20506977492570877\n",
      "Epoch 16/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.20506977492570877\n",
      "Epoch 17/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.20033959863583248\n",
      "Epoch 18/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.20033959863583248\n",
      "Epoch 19/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.20033959863583248\n",
      "Epoch 20/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.20033959863583248\n",
      "Epoch 21/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.20033959863583248\n",
      "Epoch 22/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.20033959863583248\n",
      "Epoch 23/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.20033959863583248\n",
      "Epoch 24/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.20033959863583248\n",
      "Epoch 25/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.20033959863583248\n",
      "Epoch 26/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.20033959863583248\n",
      "Epoch 27/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.20033959863583248\n"
     ]
    }
   ],
   "source": [
    "model = condensenet()\n",
    "model = model.to(device)\n",
    "\n",
    "optimizer = optim.SGD(model.parameters(), lr = 0.01, momentum = 0.9)\n",
    "\n",
    "save_file_path = 'condensenet.pth'\n",
    "\n",
    "train(model, criterion, optimizer, scheduler, save_file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 863    2   25   11    4    1   84    0    9    1\n",
      "   0  988    0    8    0    0    2    0    2    0\n",
      "  13    2  915    9   22    0   38    0    1    0\n",
      "  12    6    8  923   18    0   33    0    0    0\n",
      "   1    1   35   18  900    0   43    0    2    0\n",
      "   0    0    0    0    0  981    0   16    0    3\n",
      "  87    3   38   19   64    0  777    0   12    0\n",
      "   0    0    0    0    0    4    0  977    0   19\n",
      "   2    0    1    2    0    2    2    0  991    0\n",
      "   0    0    0    0    0    5    0   29    1  965\n",
      "Accuracy: 0.9280\n",
      "Time per Image: 0.0167\n"
     ]
    }
   ],
   "source": [
    "test(model, save_file_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 10. Random Erasing for Augmentation\n",
    "\n",
    "Add Transform from https://github.com/zhunzhong07/Random-Erasing\n",
    "\n",
    "Model : Condensenet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from random_erasing import transforms\n",
    "\n",
    "p = 0.5\n",
    "sh = 0.4\n",
    "r1 = 0.3\n",
    "\n",
    "# data augmentation\n",
    "data_transform = {\n",
    "    'train': transforms.Compose([\n",
    "        transforms.RandomHorizontalFlip(), \n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize((0.1307,), (0.3081,)),\n",
    "        transforms.RandomErasing(probability = p, sh = sh, r1 = r1, mean = [0.4914]),\n",
    "    ]),\n",
    "    'val': transforms.Compose([\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize((0.1307,), (0.3081,)),\n",
    "    ])\n",
    "}\n",
    "\n",
    "# image dataset\n",
    "image_dataset = {x: Dataset(img_array[x], label_array[x], data_transform[x]) for x in ['train', 'val']}\n",
    "\n",
    "# data loader\n",
    "data_loader = {x: torch.utils.data.DataLoader(image_dataset[x], batch_size=batch_size, shuffle=True, num_workers=4)\n",
    "                for x in ['train', 'val']}\n",
    "\n",
    "dataset_size = {x: len(image_dataset[x]) for x in ['train', 'val']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.49947190219163895\n",
      "Epoch 2/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.3785525686542193\n",
      "Epoch 3/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.29215894744793575\n",
      "Epoch 4/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.27510109134515126\n",
      "Epoch 5/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.27333004744847617\n",
      "Epoch 6/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.26767446305354436\n",
      "Epoch 7/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.26767446305354436\n",
      "Epoch 8/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.25031330053011575\n",
      "Epoch 9/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.22323818025986353\n",
      "Epoch 10/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.22323818025986353\n",
      "Epoch 11/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.21881425712505975\n",
      "Epoch 12/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.21881425712505975\n",
      "Epoch 13/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.2065012404123942\n",
      "Epoch 14/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.2065012404123942\n",
      "Epoch 15/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.2065012404123942\n",
      "Epoch 16/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.20121304468313853\n",
      "Epoch 17/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.20121304468313853\n",
      "Epoch 18/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.20121304468313853\n",
      "Epoch 19/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.20121304468313853\n",
      "Epoch 20/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.20121304468313853\n",
      "Epoch 21/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.19911201117436092\n",
      "Epoch 22/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.19821112670501073\n",
      "Epoch 23/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.19821112670501073\n",
      "Epoch 24/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.19821112670501073\n",
      "Epoch 25/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.19821112670501073\n",
      "Epoch 26/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.19821112670501073\n",
      "Epoch 27/100\n",
      "Learning Rate: 0.01\n",
      "Best Validation Loss: 0.19821112670501073\n",
      "Epoch 28/100\n",
      "Learning Rate: 0.001\n",
      "Best Validation Loss: 0.19821112670501073\n",
      "Epoch 29/100\n",
      "Learning Rate: 0.001\n",
      "Best Validation Loss: 0.18618906235694885\n",
      "Epoch 30/100\n",
      "Learning Rate: 0.001\n",
      "Best Validation Loss: 0.18618906235694885\n",
      "Epoch 31/100\n",
      "Learning Rate: 0.001\n",
      "Best Validation Loss: 0.18618906235694885\n",
      "Epoch 32/100\n",
      "Learning Rate: 0.001\n",
      "Best Validation Loss: 0.18618906235694885\n",
      "Epoch 33/100\n",
      "Learning Rate: 0.001\n",
      "Best Validation Loss: 0.18618906235694885\n",
      "Epoch 34/100\n",
      "Learning Rate: 0.001\n",
      "Best Validation Loss: 0.18618906235694885\n",
      "Epoch 35/100\n",
      "Learning Rate: 0.0001\n",
      "Best Validation Loss: 0.18618906235694885\n",
      "Epoch 36/100\n",
      "Learning Rate: 0.0001\n",
      "Best Validation Loss: 0.18618906235694885\n",
      "Epoch 37/100\n",
      "Learning Rate: 0.0001\n",
      "Best Validation Loss: 0.18618906235694885\n",
      "Epoch 38/100\n",
      "Learning Rate: 0.0001\n",
      "Best Validation Loss: 0.18618906235694885\n",
      "Epoch 39/100\n",
      "Learning Rate: 0.0001\n",
      "Best Validation Loss: 0.18618906235694885\n"
     ]
    }
   ],
   "source": [
    "model = condensenet()\n",
    "model = model.to(device)\n",
    "\n",
    "optimizer = optim.SGD(model.parameters(), lr = 0.01, momentum = 0.9)\n",
    "scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=5)\n",
    "\n",
    "save_file_path = 'condensenet_re.pth'\n",
    "\n",
    "train(model, criterion, optimizer, scheduler, save_file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 893    2   13    8    3    0   76    0    4    1\n",
      "   0  992    0    6    0    0    0    0    1    1\n",
      "  18    1  900    7   34    0   39    0    1    0\n",
      "   9    2    6  944   22    1   16    0    0    0\n",
      "   1    1   18   12  931    0   37    0    0    0\n",
      "   0    0    0    0    0  991    0    9    0    0\n",
      "  77    1   34   20   61    0  802    0    5    0\n",
      "   0    0    0    0    0    4    0  985    0   11\n",
      "   1    1    1    2    0    1    2    0  992    0\n",
      "   0    0    0    0    0    7    0   35    0  958\n",
      "Accuracy: 0.9388\n",
      "Time per Image: 0.0183\n"
     ]
    }
   ],
   "source": [
    "test(model, save_file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}