Practise Notebook_basic.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision\n",
    "import numpy as np\n",
    "import torchvision.transforms as transforms\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60000\n",
      "600\n",
      "Epoch [1/5], Iteration [100/600], Loss: 2.2360\n",
      "Epoch [1/5], Iteration [200/600], Loss: 2.1408\n",
      "Epoch [1/5], Iteration [300/600], Loss: 2.0421\n",
      "Epoch [1/5], Iteration [400/600], Loss: 1.9860\n",
      "Epoch [1/5], Iteration [500/600], Loss: 1.8835\n",
      "Epoch [1/5], Iteration [600/600], Loss: 1.7920\n",
      "Epoch [2/5], Iteration [100/600], Loss: 1.7255\n",
      "Epoch [2/5], Iteration [200/600], Loss: 1.6768\n",
      "Epoch [2/5], Iteration [300/600], Loss: 1.6237\n",
      "Epoch [2/5], Iteration [400/600], Loss: 1.5295\n",
      "Epoch [2/5], Iteration [500/600], Loss: 1.5145\n",
      "Epoch [2/5], Iteration [600/600], Loss: 1.4964\n",
      "Epoch [3/5], Iteration [100/600], Loss: 1.5087\n",
      "Epoch [3/5], Iteration [200/600], Loss: 1.3562\n",
      "Epoch [3/5], Iteration [300/600], Loss: 1.3707\n",
      "Epoch [3/5], Iteration [400/600], Loss: 1.3737\n",
      "Epoch [3/5], Iteration [500/600], Loss: 1.3118\n",
      "Epoch [3/5], Iteration [600/600], Loss: 1.3638\n",
      "Epoch [4/5], Iteration [100/600], Loss: 1.2265\n",
      "Epoch [4/5], Iteration [200/600], Loss: 1.0935\n",
      "Epoch [4/5], Iteration [300/600], Loss: 1.2359\n",
      "Epoch [4/5], Iteration [400/600], Loss: 1.1087\n",
      "Epoch [4/5], Iteration [500/600], Loss: 1.1397\n",
      "Epoch [4/5], Iteration [600/600], Loss: 1.0544\n",
      "Epoch [5/5], Iteration [100/600], Loss: 1.1278\n",
      "Epoch [5/5], Iteration [200/600], Loss: 1.0547\n",
      "Epoch [5/5], Iteration [300/600], Loss: 1.0745\n",
      "Epoch [5/5], Iteration [400/600], Loss: 1.0204\n",
      "Epoch [5/5], Iteration [500/600], Loss: 1.0686\n",
      "Epoch [5/5], Iteration [600/600], Loss: 1.1287\n",
      "Accuracy of the model on the 10000 test images: 82 %\n"
     ]
    }
   ],
   "source": [
    "input_size = 784\n",
    "output_size = 10\n",
    "num_epochs = 5\n",
    "batch_size = 100\n",
    "learning_rate = 0.001\n",
    "\n",
    "train_dataset = torchvision.datasets.MNIST(root='../ai61002s19/MNIST', train = True, transform = transforms.ToTensor(), download = False)\n",
    "test_dataset = torchvision.datasets.MNIST(root='../ai61002s19/MNIST', train = False, transform= transforms.ToTensor())\n",
    "train_loader = torch.utils.data.DataLoader(dataset = train_dataset, batch_size = batch_size, shuffle = True)\n",
    "test_loader = torch.utils.data.DataLoader(dataset = test_dataset, batch_size = batch_size, shuffle = False)                                          \n",
    "\n",
    "model = nn.Linear(input_size, output_size)\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr = learning_rate)\n",
    "\n",
    "print(len(train_dataset))\n",
    "print(len(train_loader))\n",
    "\n",
    "total_step = len(train_loader)\n",
    "for epoch in range(num_epochs):\n",
    "    for i, (images, labels) in enumerate(train_loader):\n",
    "        images = images.reshape(-1, 28*28)\n",
    "        outputs = model(images)\n",
    "        loss = criterion(outputs, labels)\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        if (i+1) % 100 == 0:\n",
    "            print ('Epoch [{}/{}], Iteration [{}/{}], Loss: {:.4f}' \n",
    "                   .format(epoch+1, num_epochs, i+1, total_step, loss.item()))\n",
    "\n",
    "with torch.no_grad():\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for images, labels in test_loader:\n",
    "        images = images.reshape(-1, 28*28)\n",
    "        outputs = model(images)\n",
    "        _, predicted = torch.max(outputs.data, 1) #return max value and the indices\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted==labels).sum()\n",
    "    print('Accuracy of the model on the 10000 test images: {} %'.format(100 * correct / total)) \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1/5], Iteration [100/600], Loss: 0.5375\n",
      "Epoch [1/5], Iteration [200/600], Loss: 0.3712\n",
      "Epoch [1/5], Iteration [300/600], Loss: 0.2424\n",
      "Epoch [1/5], Iteration [400/600], Loss: 0.4327\n",
      "Epoch [1/5], Iteration [500/600], Loss: 0.2944\n",
      "Epoch [1/5], Iteration [600/600], Loss: 0.3506\n",
      "Epoch [2/5], Iteration [100/600], Loss: 0.1986\n",
      "Epoch [2/5], Iteration [200/600], Loss: 0.1941\n",
      "Epoch [2/5], Iteration [300/600], Loss: 0.2853\n",
      "Epoch [2/5], Iteration [400/600], Loss: 0.1549\n",
      "Epoch [2/5], Iteration [500/600], Loss: 0.1121\n",
      "Epoch [2/5], Iteration [600/600], Loss: 0.1507\n",
      "Epoch [3/5], Iteration [100/600], Loss: 0.0889\n",
      "Epoch [3/5], Iteration [200/600], Loss: 0.1409\n",
      "Epoch [3/5], Iteration [300/600], Loss: 0.0846\n",
      "Epoch [3/5], Iteration [400/600], Loss: 0.0784\n",
      "Epoch [3/5], Iteration [500/600], Loss: 0.0881\n",
      "Epoch [3/5], Iteration [600/600], Loss: 0.2132\n",
      "Epoch [4/5], Iteration [100/600], Loss: 0.1325\n",
      "Epoch [4/5], Iteration [200/600], Loss: 0.0613\n",
      "Epoch [4/5], Iteration [300/600], Loss: 0.1321\n",
      "Epoch [4/5], Iteration [400/600], Loss: 0.1268\n",
      "Epoch [4/5], Iteration [500/600], Loss: 0.1063\n",
      "Epoch [4/5], Iteration [600/600], Loss: 0.1586\n",
      "Epoch [5/5], Iteration [100/600], Loss: 0.1263\n",
      "Epoch [5/5], Iteration [200/600], Loss: 0.1355\n",
      "Epoch [5/5], Iteration [300/600], Loss: 0.0722\n",
      "Epoch [5/5], Iteration [400/600], Loss: 0.0694\n",
      "Epoch [5/5], Iteration [500/600], Loss: 0.0414\n",
      "Epoch [5/5], Iteration [600/600], Loss: 0.0394\n",
      "Accuracy of the network on the 10000 test images: 97 %\n"
     ]
    }
   ],
   "source": [
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "input_size = 784\n",
    "hidden_size = 100\n",
    "num_classes = 10\n",
    "num_epochs = 5 \n",
    "batch_size = 100\n",
    "learning_rate = 0.001\n",
    "\n",
    "train_dataset = torchvision.datasets.MNIST(root = '../ai61002s19/MNIST', train = True, transform = transforms.ToTensor(), download = False)\n",
    "test_dataset = torchvision.datasets.MNIST(root = '../ai61002s19/MNIST', train = False, transform = transforms.ToTensor())\n",
    "train_loader = torch.utils.data.DataLoader(dataset = train_dataset, batch_size = batch_size, shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(dataset = test_dataset, batch_size = batch_size, shuffle=False)\n",
    "\n",
    "class NeuralNet(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, num_classes):\n",
    "        super(NeuralNet, self).__init__()\n",
    "        self.fc1 = nn.Linear(input_size, hidden_size)\n",
    "        self.relu = nn.ReLU()\n",
    "        self.fc2 = nn.Linear(hidden_size, num_classes)\n",
    "    def forward(self, x):\n",
    "        out = self.fc1(x)\n",
    "        out = self.relu(out)\n",
    "        out = self.fc2(out)\n",
    "        return out\n",
    "    \n",
    "model = NeuralNet(input_size, hidden_size, num_classes).to(device)\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(),lr = learning_rate)\n",
    "\n",
    "total_step = len(train_loader)\n",
    "for epoch in range(num_epochs):\n",
    "    for i, (images, labels) in enumerate(train_loader):\n",
    "        images = images.reshape(-1,28*28).to(device)\n",
    "        labels = labels.to(device)\n",
    "        outputs = model(images)\n",
    "        loss = criterion(outputs, labels)\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        if (i+1) % 100 == 0:\n",
    "            print ('Epoch [{}/{}], Iteration [{}/{}], Loss: {:.4f}' \n",
    "                   .format(epoch+1, num_epochs, i+1, total_step, loss.item()))\n",
    "\n",
    "with torch.no_grad():\n",
    "    correct = 0 \n",
    "    total = 0\n",
    "    for images, labels in (test_loader):\n",
    "        images= images.reshape(-1, 28*28).to(device)\n",
    "        labels = labels.to(device)\n",
    "        outputs = model(images)\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        total+=predicted.size(0)\n",
    "        correct += (predicted==labels).sum()\n",
    "        \n",
    "    print('Accuracy of the network on the 10000 test images: {} %'.format(100 * correct / total))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'torch' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-7cd4f2fb6b3b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdevice\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'cuda:0'\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_available\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m'cpu'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mnum_epochs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m5\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mnum_classes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mbatch_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m100\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'torch' is not defined"
     ]
    }
   ],
   "source": [
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "num_epochs = 5\n",
    "num_classes = 10\n",
    "batch_size = 100 \n",
    "learning_rate = 0.001\n",
    "\n",
    "class Conv(nn.Module):\n",
    "    def __init__(self, num_classes):\n",
    "        super(Conv, self).__init__()\n",
    "        self.layer1 = nn.Sequential(\n",
    "            nn.Conv2d(1,16, kernel_size=5, stride = 1, padding = 2),\n",
    "            nn.BatchNorm2d(16),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size = 2, stride=2))\n",
    "        self.layer2 = nn.Sequential(\n",
    "            nn.Conv2d(16,32,kernel_size = 5, padding =2, stride =1),\n",
    "            nn.BatchNorm2d(32),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size = 2, stride =2))\n",
    "        self.fc = nn.Linear(7*7*32, num_classes)\n",
    "    def forward(self, x):\n",
    "        out = self.layer1(x)\n",
    "        out = self.layer2(out)\n",
    "#         print(out.size(0))\n",
    "        out = out.reshape(out.size(0), -1)#here we provide a batch of image so out.size(0) is the batch_size\n",
    "        out = self.fc(out)\n",
    "        return out\n",
    "train_dataset = torchvision.datasets.MNIST(root='../ai61002s19/MNIST', train=True, transform = transforms.ToTensor(), download = False)\n",
    "test_dataset = torchvision.datasets.MNIST(root='../ai61002s19/MNIST', train=False, transform = transforms.ToTensor(), download = False)\n",
    "train_loader = torch.utils.data.DataLoader(dataset = train_dataset, batch_size = batch_size, shuffle = True)\n",
    "test_loader = torch.utils.data.DataLoader(dataset = test_dataset, batch_size = batch_size, shuffle = True)\n",
    "\n",
    "model = Conv(num_classes)#.to(device)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr= learning_rate)\n",
    "# out = model(train_dataset[0][0].reshape(1,1,28,28))\n",
    "\n",
    "train_step = len(train_loader)\n",
    "for epoch in range(num_epochs):\n",
    "    for i, (images, labels) in enumerate(train_loader):\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "#         print(images.shape)\n",
    "        outputs = model(images)\n",
    "        loss = criterion(outputs, labels)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        if (i+1) % 100 == 0:\n",
    "            print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}' \n",
    "                   .format(epoch+1, num_epochs, i+1, total_step, loss.item()))\n",
    "\n",
    "model.eval()  #it is typically used for batchnorm so that it uses moving variance and mean instead of the whole batch\n",
    "with torch.no_grad():\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for images, labels in test_loader:\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "        outputs = model(images)\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted == labels).sum().item()\n",
    "\n",
    "    print('Test Accuracy of the model on the 10000 test images: {} %'.format(100 * correct / total))      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Epoch [1/30], Step [100/500] Loss: 1.6097\n",
      "Epoch [1/30], Step [200/500] Loss: 1.4205\n",
      "Epoch [1/30], Step [300/500] Loss: 1.4091\n",
      "Epoch [1/30], Step [400/500] Loss: 1.2042\n",
      "Epoch [1/30], Step [500/500] Loss: 1.0310\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-14-407b12263122>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    101\u001b[0m         \u001b[0;31m# Backward and optimize\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    102\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 103\u001b[0;31m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    104\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    105\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m     91\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m         \"\"\"\n\u001b[0;32m---> 93\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m     88\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m     89\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 90\u001b[0;31m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m     91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "batch_size = 100\n",
    "num_epochs = 30\n",
    "learning_rate = 0.001\n",
    "transform = transforms.Compose([\n",
    "    transforms.Pad(4),\n",
    "    transforms.RandomHorizontalFlip(), \n",
    "    transforms.RandomCrop(32), \n",
    "    transforms.ToTensor()])\n",
    "\n",
    "train_dataset = torchvision.datasets.CIFAR10(root='../ai61002s19/', train=True, transform = transform, download = True)\n",
    "test_dataset = torchvision.datasets.CIFAR10(root='../ai61002s19/', train=False, transform = transforms.ToTensor(), download = False)\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size = batch_size, shuffle= True)\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size = batch_size, shuffle= False)\n",
    "\n",
    "class ResidualBlock(nn.Module):\n",
    "    def __init__(self, in_channel, out_channel,stride=1, downsample=None):\n",
    "        super(ResidualBlock, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(in_channel, out_channel, kernel_size = 3, stride = stride, padding=1)\n",
    "        self.bn1 = nn.BatchNorm2d(out_channel)\n",
    "        self.relu = nn.ReLU()\n",
    "        self.conv2 = nn.Conv2d(out_channel, out_channel, kernel_size = 3, stride = 1, padding=1)  #we change the size only once\n",
    "        self.bn2 = nn.BatchNorm2d(out_channel)\n",
    "        self.downsample = downsample\n",
    "    def forward(self,x):\n",
    "        residual = x\n",
    "        out = self.conv1(x)\n",
    "        out = self.bn1(out)\n",
    "        out = self.relu(out)\n",
    "        out = self.conv2(out)\n",
    "        out = self.bn2(out)\n",
    "        if self.downsample:#to be used when input size does not match output size\n",
    "            residual = self.downsample(x)\n",
    "        out += residual\n",
    "        out = self.relu(out)\n",
    "        return(out)\n",
    "\n",
    "class ResNet(nn.Module):\n",
    "    def __init__(self, block, num_classes=10):\n",
    "        super(ResNet, self).__init__()\n",
    "        self.in_channel = 16\n",
    "        self.conv1 = nn.Conv2d(3,16, stride =1, kernel_size = 3, padding = 1)\n",
    "        self.bn = nn.BatchNorm2d(16)\n",
    "        self.relu = nn.ReLU()\n",
    "        self.block1 = self.make_layer(block, 16, 1)\n",
    "        self.block2 = self.make_layer(block, 16, 1)\n",
    "        self.block3 = self.make_layer(block, 32, 2)\n",
    "        self.block4 = self.make_layer(block, 32, 1)\n",
    "        self.block5 = self.make_layer(block, 64, 2)\n",
    "        self.block6 = self.make_layer(block, 64, 1)\n",
    "        self.avg_pool = nn.AvgPool2d(8) #8 is the kernel size so it is taking average of 8x8\n",
    "        self.fc = nn.Linear(64, num_classes)\n",
    "    def make_layer(self, block, out_channel, stride=1):\n",
    "        downsample = None\n",
    "        if(stride!=1) or (self.in_channel != out_channel):#input size not equal to output size only when stride not 1 or input channel and output channel are not same \n",
    "            downsample = nn.Sequential(\n",
    "            nn.Conv2d(self.in_channel, out_channel, kernel_size = 3, stride = stride, padding = 1),\n",
    "            nn.BatchNorm2d(out_channel))\n",
    "        out_layer = block(self.in_channel, out_channel, stride, downsample)\n",
    "        self.in_channel = out_channel\n",
    "        return(out_layer)\n",
    "    def forward(self,x):\n",
    "        out = self.conv1(x)\n",
    "        out = self.bn(out)\n",
    "        out = self.relu(out)\n",
    "        out = self.block1(out)\n",
    "        out = self.block2(out)\n",
    "        out = self.block3(out)\n",
    "        out = self.block4(out)\n",
    "        out = self.block5(out)\n",
    "        out = self.block6(out)\n",
    "        out = self.avg_pool(out)\n",
    "        out = out.view(out.size(0), -1)\n",
    "        out = self.fc(out)\n",
    "        return out\n",
    "model = ResNet(ResidualBlock).to(device)\n",
    "# print(train_dataset[0][0].shape)\n",
    "# out = model(train_dataset[0][0].reshape(1,3,32,32))\n",
    "\n",
    "# Loss and optimizer\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "# For updating learning rate\n",
    "def update_lr(optimizer, lr):    \n",
    "    for param_group in optimizer.param_groups:\n",
    "        param_group['lr'] = lr\n",
    "\n",
    "# Train the model\n",
    "total_step = len(train_loader)\n",
    "curr_lr = learning_rate\n",
    "for epoch in range(num_epochs):\n",
    "    for i, (images, labels) in enumerate(train_loader):\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "        \n",
    "        # Forward pass\n",
    "        outputs = model(images)\n",
    "        loss = criterion(outputs, labels)\n",
    "        \n",
    "        # Backward and optimize\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        if (i+1) % 100 == 0:\n",
    "            print (\"Epoch [{}/{}], Step [{}/{}] Loss: {:.4f}\"\n",
    "                   .format(epoch+1, num_epochs, i+1, total_step, loss.item()))\n",
    "\n",
    "    # Decay learning rate\n",
    "    if (epoch+1) % 20 == 0:\n",
    "        curr_lr /= 3\n",
    "        update_lr(optimizer, curr_lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}