U47768_CH04_ModelFitting.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/submouse9903/uos-deepLearning/blob/main/U47768_CH04_ModelFitting.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FKK8Kf8u0v1H"
      },
      "outputs": [],
      "source": [
        "import torch \n",
        "import torch.nn as nn\n",
        "import torchvision"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torchvision.transforms as transforms\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np"
      ],
      "metadata": {
        "id": "CDd2gsylKKAt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YBqeX_n01rLS",
        "outputId": "5ca2f6c9-28ac-4f18-c46b-315443addaef"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to ./data/MNIST/raw/train-images-idx3-ubyte.gz\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 9912422/9912422 [00:00<00:00, 60987744.34it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting ./data/MNIST/raw/train-images-idx3-ubyte.gz to ./data/MNIST/raw\n",
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to ./data/MNIST/raw/train-labels-idx1-ubyte.gz\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 28881/28881 [00:00<00:00, 93109680.11it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting ./data/MNIST/raw/train-labels-idx1-ubyte.gz to ./data/MNIST/raw\n",
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to ./data/MNIST/raw/t10k-images-idx3-ubyte.gz\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1648877/1648877 [00:00<00:00, 121685811.25it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting ./data/MNIST/raw/t10k-images-idx3-ubyte.gz to ./data/MNIST/raw\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to ./data/MNIST/raw/t10k-labels-idx1-ubyte.gz\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 4542/4542 [00:00<00:00, 15928535.76it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting ./data/MNIST/raw/t10k-labels-idx1-ubyte.gz to ./data/MNIST/raw\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "train_dataset = torchvision.datasets.MNIST(root='./data', \n",
        "                                           train=True, \n",
        "                                           transform=transforms.ToTensor(),  \n",
        "                                           download=True)\n",
        "\n",
        "test_dataset = torchvision.datasets.MNIST(root='./data', \n",
        "                                          train=False, \n",
        "                                          transform=transforms.ToTensor())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "975Mm0dMk9yN",
        "outputId": "3da8bddf-a73b-400a-f213-f96ac9ad5448"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.2000, 0.6235, 0.9922,\n",
              "         0.6235, 0.1961, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.1882, 0.9333, 0.9882, 0.9882,\n",
              "         0.9882, 0.9294, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.0000, 0.0000, 0.2118, 0.8902, 0.9922, 0.9882, 0.9373,\n",
              "         0.9137, 0.9882, 0.2235, 0.0235, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.0392, 0.2353, 0.8784, 0.9882, 0.9922, 0.9882, 0.7922,\n",
              "         0.3294, 0.9882, 0.9922, 0.4784, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.6392, 0.9882, 0.9882, 0.9882, 0.9922, 0.9882, 0.9882,\n",
              "         0.3765, 0.7412, 0.9922, 0.6549, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.2000, 0.9333, 0.9922, 0.9922, 0.7451, 0.4471, 0.9922, 0.8941,\n",
              "         0.1843, 0.3098, 1.0000, 0.6588, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.1882, 0.9333, 0.9882, 0.9882, 0.7020, 0.0471, 0.2941, 0.4745, 0.0824,\n",
              "         0.0000, 0.0000, 0.9922, 0.9529, 0.1961, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.1490,\n",
              "         0.6471, 0.9922, 0.9137, 0.8157, 0.3294, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.9922, 0.9882, 0.6471, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0275, 0.6980,\n",
              "         0.9882, 0.9412, 0.2784, 0.0745, 0.1098, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.9922, 0.9882, 0.7647, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.2235, 0.9882,\n",
              "         0.9882, 0.2471, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.9922, 0.9882, 0.7647, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.7765, 0.9922,\n",
              "         0.7451, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 1.0000, 0.9922, 0.7686, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.2980, 0.9647, 0.9882,\n",
              "         0.4392, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.9922, 0.9882, 0.5804, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.3333, 0.9882, 0.9020,\n",
              "         0.0980, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0275, 0.5294, 0.9922, 0.7294, 0.0471, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.3333, 0.9882, 0.8745,\n",
              "         0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0275,\n",
              "         0.5137, 0.9882, 0.8824, 0.2784, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.3333, 0.9882, 0.5686,\n",
              "         0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.1882, 0.6471,\n",
              "         0.9882, 0.6784, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.3373, 0.9922, 0.8824,\n",
              "         0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.4471, 0.9333, 0.9922,\n",
              "         0.6353, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.3333, 0.9882, 0.9765,\n",
              "         0.5725, 0.1882, 0.1137, 0.3333, 0.6980, 0.8824, 0.9922, 0.8745, 0.6549,\n",
              "         0.2196, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.3333, 0.9882, 0.9882,\n",
              "         0.9882, 0.8980, 0.8431, 0.9882, 0.9882, 0.9882, 0.7686, 0.5098, 0.0000,\n",
              "         0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.1098, 0.7804, 0.9882,\n",
              "         0.9882, 0.9922, 0.9882, 0.9882, 0.9137, 0.5686, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0980, 0.5020,\n",
              "         0.9882, 0.9922, 0.9882, 0.5529, 0.1451, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000],\n",
              "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
              "         0.0000]])"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ],
      "source": [
        "#train_dataset[1][0][0,:,:]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 189
        },
        "id": "7r0fNFvW1-nH",
        "outputId": "9b1c0c5a-5e8f-45b7-c158-018060131bf0"
      },
      "outputs": [
        {
          "output_type": "error",
          "ename": "IndexError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-5-bfd470556814>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_dataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcmap\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_cmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'gray'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m \u001b[0mtrain_dataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mIndexError\u001b[0m: index 1 is out of bounds for dimension 0 with size 1"
          ]
        }
      ],
      "source": [
        "plt.imshow(train_dataset[1][0][1,:,:], cmap=plt.get_cmap('gray'))\n",
        "print( train_dataset[1][0] )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XvtvEO2s4E5t",
        "outputId": "efc0f8fe-7aad-47ae-8280-1a4b4d7a1cc9"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([1, 28, 28])"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ],
      "source": [
        "train_dataset[1][0].shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0LK4-vC7EiSB",
        "outputId": "9721edba-062b-478d-87a7-8d984d035679"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([784])"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ],
      "source": [
        "train_dataset[0][0].reshape(-1).shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oa-AIEF8yYl7",
        "outputId": "8aed4560-f145-4b0d-bee5-84548a8e852f"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([60000, 28, 28])"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ],
      "source": [
        "rx = train_dataset.data\n",
        "ry = train_dataset.targets\n",
        "rx.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AJDYntmE5X4i",
        "outputId": "67c2acc4-3908-4e50-a6a2-5f2fc01e15b7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "x dtype: torch.float32\n",
            "x shape: torch.Size([50000, 784])\n",
            "y dtype: torch.int64\n"
          ]
        }
      ],
      "source": [
        "n = 50000\n",
        "p = 28*28\n",
        "x = rx[0:n]\n",
        "y = ry[0:n]\n",
        "x = x.reshape(n,-1)\n",
        "x = x.float()\n",
        "print(\"x dtype:\", x.dtype)\n",
        "print(\"x shape:\", x.shape)\n",
        "print(\"y dtype:\", y.dtype)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ryg8spZR00Vc"
      },
      "outputs": [],
      "source": [
        "h = 64\n",
        "output_dim = 10"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "g6KK9GB63YAy"
      },
      "outputs": [],
      "source": [
        "model = nn.Sequential(nn.Linear(p,h),\n",
        "                      nn.ReLU(),\n",
        "                      nn.Linear(h, h),\n",
        "                      nn.ReLU(),\n",
        "                      nn.Linear(h, h),\n",
        "                      nn.ReLU(),\n",
        "                      nn.Linear(h, output_dim),\n",
        "                      nn.Sigmoid(),\n",
        "                      nn.Softmax())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MjD1wOkVpSDz",
        "outputId": "28c7e6cf-2309-4e44-a94b-548a0739c3d6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sequential(\n",
            "  (0): Linear(in_features=784, out_features=64, bias=True)\n",
            "  (1): ReLU()\n",
            "  (2): Linear(in_features=64, out_features=64, bias=True)\n",
            "  (3): ReLU()\n",
            "  (4): Linear(in_features=64, out_features=64, bias=True)\n",
            "  (5): ReLU()\n",
            "  (6): Linear(in_features=64, out_features=10, bias=True)\n",
            "  (7): Sigmoid()\n",
            "  (8): Softmax(dim=None)\n",
            ")\n"
          ]
        }
      ],
      "source": [
        "print(model)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0tbFueiDHdvz"
      },
      "outputs": [],
      "source": [
        "learning_rate = 0.005\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
        "cross_entropy = nn.CrossEntropyLoss()  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KGz0GWihH-Ux",
        "outputId": "0ecb40fe-7b82-4d50-8724-e28f36c4afa8"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([0.1299, 0.1433, 0.0926, 0.0575, 0.1390, 0.0888, 0.1494, 0.0720, 0.0667,\n",
              "        0.0607], grad_fn=<SliceBackward0>)"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ],
      "source": [
        "output = model(x)\n",
        "output[0,:]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DXDeGAq66D52",
        "outputId": "f4ffa6d8-b631-4eba-feec-e82c9187aac9"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([50000, 10])"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ],
      "source": [
        "output.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 445
        },
        "id": "ci7hNgG8HOvq",
        "outputId": "3ac489c6-bbbd-4950-e558-0800f5f0862d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy: 0.1335\n",
            "Accuracy: 0.8354\n",
            "Accuracy: 0.8640\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-19-d668b96217f8>\u001b[0m in \u001b[0;36m<cell line: 4>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnum_B\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m   \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m   \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcross_entropy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m   \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1499\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1500\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1502\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1503\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    215\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    216\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 217\u001b[0;31m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    218\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    219\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1499\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1500\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1502\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1503\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/linear.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    112\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    113\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 114\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    115\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    116\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mextra_repr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ],
      "source": [
        "num_B = 300\n",
        "loss_vec = np.zeros(num_B)\n",
        "optimizer.zero_grad()\n",
        "for i in range(num_B):\n",
        "  output = model(x)\n",
        "  loss = cross_entropy(output, y)\n",
        "  optimizer.zero_grad()\n",
        "  loss.backward()\n",
        "  optimizer.step()\n",
        "  predicted = torch.max(output.data,1)\n",
        "  v =((predicted.indices == y).sum()/n ).item()\n",
        "  loss_vec[i] = v\n",
        "  if i%30 == 0:\n",
        "    print (\"Accuracy: {:.4f}\".format(v))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7q4pz4rsHbh_",
        "outputId": "f5cc252d-ade5-4c0a-ccd4-d7ef167d4d27"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "x dtype: torch.float32\n",
            "x shape: torch.Size([10000, 784])\n",
            "y dtype: torch.int64\n"
          ]
        }
      ],
      "source": [
        "n = 10000\n",
        "p = 28*28\n",
        "x_test = rx[50000:60000]\n",
        "y_test = ry[50000:60000]\n",
        "x_test = x_test.reshape(n,-1)\n",
        "x_test = x_test.float()\n",
        "print(\"x dtype:\", x_test.dtype)\n",
        "print(\"x shape:\", x_test.shape)\n",
        "print(\"y dtype:\", y_test.dtype)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "u4bcwXQWNKID",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5548af10-9a21-4de6-f5a6-7e6bbc294d36"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/container.py:217: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
            "  input = module(input)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9544000029563904"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ],
      "source": [
        "output = model(x_test)\n",
        "predicted = torch.max(output.data,1)\n",
        "v =((predicted.indices == y_test).sum()/n ).item()\n",
        "v"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "umc5LfXqrefl"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}