04_model_loss_practice.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "04_model_loss_practice.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "1_1RVWc42O_1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3e32045a-a9e0-4b9b-a8d5-c8c41f744df3"
      },
      "source": [
        "!pip install -Uqq fastbook\n",
        "import fastbook\n",
        "fastbook.setup_book()"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 727kB 7.4MB/s \n",
            "\u001b[K     |████████████████████████████████| 51kB 7.6MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.2MB 16.1MB/s \n",
            "\u001b[K     |████████████████████████████████| 204kB 37.4MB/s \n",
            "\u001b[K     |████████████████████████████████| 61kB 8.8MB/s \n",
            "\u001b[K     |████████████████████████████████| 61kB 7.5MB/s \n",
            "\u001b[?25hMounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l6dvmqS_2aaP"
      },
      "source": [
        "from fastai.vision.all import *\n",
        "from fastbook import *\n",
        "\n",
        "matplotlib.rc('image', cmap='Greys')"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "id": "VZmkljEB2czH",
        "outputId": "5499b057-f913-4915-da69-d95dc614ac99"
      },
      "source": [
        "path = untar_data(URLs.MNIST_SAMPLE)\n",
        "Path.BASE_PATH = path\n",
        "threes = (path/'train'/'3').ls().sorted()\n",
        "sevens = (path/'train'/'7').ls().sorted()\n",
        "threes"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(#6131) [Path('train/3/10.png'),Path('train/3/10000.png'),Path('train/3/10011.png'),Path('train/3/10031.png'),Path('train/3/10034.png'),Path('train/3/10042.png'),Path('train/3/10052.png'),Path('train/3/1007.png'),Path('train/3/10074.png'),Path('train/3/10091.png')...]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 45
        },
        "id": "SHWkoc6V3CSL",
        "outputId": "c64313bf-10aa-4289-dbff-9076689e3c48"
      },
      "source": [
        "im3_path = threes[1]\n",
        "im3 = Image.open(im3_path)\n",
        "im3"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAAA9ElEQVR4nM3Or0sDcRjH8c/pgrfBVBjCgibThiKIyTWbWF1bORhGwxARxH/AbtW0JoIGwzXRYhJhtuFY2q1ocLgbe3sGReTuuWbwkx6+r+/zQ/pncX6q+YOldSe6nG3dn8U/rTQ70L8FCGJUewvxl7NTmezNb8xIkvKugr1HSeMP6SrWOVkoTEuSyh0Gm2n3hQyObMnXnxkempRrvgD+gokzwxFAr7U7YXHZ8x4A/Dl7rbu6D2yl3etcw/F3nZgfRVI7rXM7hMUUqzzBec427x26rkmlkzEEa4nnRqnSOH2F0UUx0ePzlbuqMXAHgN6GY9if5xP8dmtHFfwjuQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<PIL.PngImagePlugin.PngImageFile image mode=L size=28x28 at 0x7F180A1B5D90>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MjoIFFXF3VXD",
        "outputId": "6b2251bd-dc00-4d42-8128-4653cb1c1ad7"
      },
      "source": [
        "# get the validaition data\n",
        "threes_valid = (path/'valid'/'3').ls().sorted()\n",
        "sevens_valid = (path/'valid'/'7').ls().sorted()\n",
        "threes_valid"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(#1010) [Path('valid/3/1020.png'),Path('valid/3/1028.png'),Path('valid/3/1042.png'),Path('valid/3/1062.png'),Path('valid/3/1066.png'),Path('valid/3/1067.png'),Path('valid/3/1069.png'),Path('valid/3/1072.png'),Path('valid/3/1092.png'),Path('valid/3/1095.png')...]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cIh1jOHy4Ibc"
      },
      "source": [
        "path_to_tensor = lambda file_list: torch.stack([tensor(Image.open(img_path)) for img_path in file_list]).float()/255"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "id": "PgjLCfba4h5z",
        "outputId": "e95298e3-d752-47e2-d15f-164bab88fdc1"
      },
      "source": [
        "train_sevens = path_to_tensor(sevens)\n",
        "train_sevens.shape\n",
        "show_image(train_sevens[0])"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7f180a094f90>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEQAAABECAYAAAA4E5OyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAGDElEQVR4nO2awU4TWxiAv9N2KNNaWjpNqlCFKiA2gkIUE40xJiyMC1dufA+Xrn0G38CNunKjcWUQI4nCQmJaaYRAkQrVNoVoO9POXdzM3HZs4VJbOrl3vqQLeuicM1//85/zz6nQdR2Hf3B1ewB2wxFiwRFiwRFiwRFiwXNA+395CRKN3nQixIIjxIIjxIIjxIIjxIIjxIIjxIIjxIIjxIIjxIIjxIIjxMJBxV1TdF2nWq2i6zqaplEul9na2kJVVX79+tXOMTbF5XJx/Phx/H4/Pp8Pj6fl2zH5IyHlchlN0ygWi6RSKR4+fEgmk2FtbQ1d1xGiYUFJtVpFCNGwXdd1dF3H5WocvEa7EILe3l7u37/P1atXmZ6eJhAItHo7JocWYkRGqVRifX2dQqHAysoKqVSK9fV18vk8P3/+BNj3hg8S0qy9Wq3WXVtVVSqVymFvoyktCSmXy+zs7PD48WPS6TTPnz+nVCpRqVTMb7920LUIIfaNEINmEeJyucwI2u/zrdKSEFVVKRaLrKyskE6nKZVKaJpW93+yLBMOh81B7+3tUSgU6qZSswhp1m4cmQghCIfDBINBBgYGCIfDuN3uw95KQw4tpFqtsru7y9bWFq9eveLHjx80OtsJBoNMT0+b3/SXL18oFAp/PmL+FhKPx4nH4yQSCQYHB9uSUKEFIUIIfD4fiqJw7do10uk0yWTSnMexWIybN28SiUQYHh5GCIGu6+RyOS5fvlyXI5pR2766ukomkyGbzZpChRCEQiEzSnw+X/cixO124/f7icVi3L59m48fP5JOp00hly5d4sGDB/h8PmRZNj9XLpdbWo5fvnzJu3fvmJubq4swRVE4ceIE4XAYv99/6Os2o6U4c7lcyLLMlStXOHPmDNFoFFVVAUgkEoTDYTweT10YS5KE1+s98NpGBG1vb5PL5fj8+TPJZJJ8Pm/2LUkSsizT19fXtsgwaGnKGFFy/vx5AG7cuFHX/idUKhUqlQqFQoGFhQUWFhZ48+aN2S5JEj09PfT19dHf39+23GHQlqu1c/lTVZVyucynT5+Yn59nbW2trh9DxMWLF5mZmaG3t7dtfUObhLQTVVUpFAq8f/+eZ8+eAfV7EkVRGBoaYmZmhrNnz7a9f9sJ2d7eZnFxkc3Nzd/ahBBMTU0xMTFBX19fR/q3XbW7sbHBixcvSKVSv7W53W4mJiaYnZ1tS93SCNsJ2dzc5MOHD2Sz2Ybtp06dYmho6F+tWK1guynz9etXlpaWGtY6QghisRiRSKRj/XddiKZpaJrG6uoqi4uLvH37lmq1ahZx8HdSvXv3LhcuXGB4eLij4+m6kEqlQqlUYm5ujkePHrGxsQHw2/b++vXr3LlzB0VROjqergvJ5XIsLy+ztLREJpNhb2/PnC77VcWdoutCdnZ2eP36NclkklwuV/esxBDRqWcfjei6kIMYHR3l9OnTjI2NcezYsaYPjtqF7ZZdK/F4nHPnzqEoCj09PY6QaDTK+Pg44XAYSZLaXt1asZUQY5mtfQIXDAYZHBxElmU8Hk/Hc4kthNSe8Riv2r+PElsIARreeDd+h2+LVca6zBrvdTqBNsI2EVK71+jWpgxsJMQu2F6IpmnmqeBR5BRb5JD9mJ+f59u3b4RCIWRZRpZlJEnqWH+2EWIss1B/ZGkUfNlsllKp1LEHQwa2ENJoD+JyuahWq+TzefNcWFXVjk8bWwippTZKhBDmzx2OQgbYREjtPsR4UmZdev835b/H4zGTpcfjqfuNiRDCPAI9ikoXbCAkGo1y69Ytdnd3WV5eJp/PUywWzfapqSnGxsZIJBIEAoH/frXr9XqJRCKcPHmSkZER+vv7kSSJUCjEwMAAIyMjjI6OEgqF8Hg8HY8ScUCi6ngWM6bI9+/fyWazPHnyhKdPn3Lv3j1mZ2dRFIVAIEAwGMTr9bYznzS8SNenjNvtxu12EwwGkSSJ8fFxJicnmZycJB6P4/P5kCQJSZKOJId0PUIMjL1IqVSiXC7j9XrrJHRgpWl4MdsI6QItTZmjrb1tQNdXGbvhCLHgCLHgCLHgCLHgCLHwF0ErgqWlqEYKAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 72x72 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lmym30mq3Kcg",
        "outputId": "a777e5ea-a096-4cef-b28a-d96de891289c"
      },
      "source": [
        "# create list tensors for training an validation sets\n",
        "train_threes = path_to_tensor(threes)\n",
        "valid_threes = path_to_tensor(threes_valid)\n",
        "valid_sevens = path_to_tensor(sevens_valid)\n",
        "valid_sevens.shape"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([1028, 28, 28])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "srArYiwZ8ik6"
      },
      "source": [
        "###**Get clear dependant/independant variable sets, a way to initialize parameters, a simple model, and a loss using sigmoid**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8PMFuK7w8cmY",
        "outputId": "350b2c7c-78ce-4152-b119-4d38f0e79c9c"
      },
      "source": [
        "# create training x and y\n",
        "train_x = torch.cat([train_threes, train_sevens]).view(-1, 28*28)\n",
        "train_x.shape"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([12396, 784])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s07dBadK9hEQ",
        "outputId": "e39f1d76-1be8-4867-b91e-f85c711f35ab"
      },
      "source": [
        "# create a list containing 1 for 3s and 0 for 7s, turn into tensor of size x 1\n",
        "train_y = tensor([1]*len(threes) + [0]*len(sevens)).unsqueeze(1)\n",
        "train_y.shape"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([12396, 1])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HKDiOPyz-f42",
        "outputId": "31764ca5-3f32-4642-e172-8535007b9253"
      },
      "source": [
        "valid_x = torch.cat([valid_threes, valid_sevens]).view(-1, 28*28)\n",
        "valid_y = tensor([1]*len(valid_threes) + [0]*len(valid_sevens)).unsqueeze(1)\n",
        "valid_x.shape, valid_y.shape"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(torch.Size([2038, 784]), torch.Size([2038, 1]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4ZtEX26R-8p8"
      },
      "source": [
        "from typing import Tuple\n",
        "# create a function to initialize parameters which require grad from pytorch\n",
        "def init_params(size: Tuple):\n",
        "  return torch.randn(size).requires_grad_()"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lEqIgUez_iVh",
        "outputId": "69effe24-0bd5-4b19-b987-2a0901a4b414"
      },
      "source": [
        "# create basic model weights\n",
        "weights = init_params((28*28, 1))\n",
        "weights.shape"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([784, 1])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ghXCNRl6_vf4",
        "outputId": "20981e96-e152-4079-8519-a23fab7a63bc"
      },
      "source": [
        "# create basic model bias\n",
        "bias = init_params((1))\n",
        "bias.shape"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([1])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q6ExH5gT_6PW"
      },
      "source": [
        "# manually implement a linear model \n",
        "def manual_linear(train_x):\n",
        "  # perform matrix multiplication between train_x and weights\n",
        "  # note shape of train_x = n x 784 and weights = 784 x 1 => matrix multiplication works with output n x 1\n",
        "  return train_x@weights + bias"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "orREK1_MCF9n",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9aace3c8-20c3-4ef2-9ac8-ae9bbd2ad510"
      },
      "source": [
        "# to make predictions using the linear model:\n",
        "preds = manual_linear(train_x)\n",
        "preds.shape"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([12396, 1])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hxBhf-a2Afd4"
      },
      "source": [
        "# create a loss function to evaluate results from the preds\n",
        "def loss(preds, target):\n",
        "  # make sure all predictions are between 0 and 1 for loss to work\n",
        "  preds = preds.sigmoid()\n",
        "\n",
        "  # make loss function for 3s and 7s\n",
        "  all_loses = torch.where(target == 1, 1-preds, preds)\n",
        "\n",
        "  # aggregate all of the losses using mean to obtain single value for loss\n",
        "  return all_loses.mean()"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NABHMuVNBeg3"
      },
      "source": [
        "# create a basic class to create a linear model\n",
        "\n",
        "class LinearModel:\n",
        "  def __init__(self, size, lr):\n",
        "    self.weights = torch.randn(size).requires_grad_()\n",
        "    self.bias = torch.randn((1)).requires_grad_()\n",
        "    self.preds = None\n",
        "    self.loss = None\n",
        "\n",
        "    self.optim = SGDOptimizer([self.weights, self.bias], lr)\n",
        "  \n",
        "  def get_params(self):\n",
        "    return (self.weights, self.bias)\n",
        "\n",
        "  def apply_linear(self, train_x):\n",
        "    self.preds = train_x@self.weights + self.bias\n",
        "    return self.preds\n",
        "  \n",
        "  def linear_loss(self, train_x, target):\n",
        "    if self.preds == None:\n",
        "      preds = self.apply_linear(train_x).sigmoid()\n",
        "    else:\n",
        "      preds = self.preds.sigmoid()\n",
        "    # loss fn where lower is better in terms of predicted probabilities\n",
        "    self.loss =  torch.where(target == 1, 1-preds, preds).mean()\n",
        "    return self.loss\n",
        "  \n",
        "  # combine the linear model, loss, and gradient calculation into single wrapper\n",
        "  def calc_grad(self, train_x, targ_y):\n",
        "    self.apply_linear(train_x)\n",
        "    self.linear_loss(None, targ_y)\n",
        "    self.loss.backward()\n",
        "  \n",
        "\n",
        "  def train_epoch(self, data_loader):\n",
        "    # loop through each batch in the data loader, use the calc grad wrapper, and take steps with the optimizer\n",
        "    for batch_x, batch_y in data_loader:\n",
        "      self.calc_grad(batch_x, batch_y)\n",
        "      self.optim.step()\n",
        "      self.optim.zero_grad()\n",
        "  \n",
        "  def batch_accuracy(self, batch_x, target_y):\n",
        "    # make predictions on the given batch\n",
        "    preds = self.apply_linear(batch_x).sigmoid()\n",
        "\n",
        "    # anything greater than zero is predicted a 3; sigmoid means that anything > 0.5 is a 3\n",
        "    # this gives a tensor of true and falses (true for correct pred, else false)\n",
        "    correct = (preds > 0.5) == target_y\n",
        "\n",
        "    # get the mean number of correct preds\n",
        "    return correct.float().mean()\n",
        "  \n",
        "  def validate_epoch(self, valid_data_loader):\n",
        "    # get the accuracy for each epoch\n",
        "    all_accs = [self.batch_accuracy(valid_x, valid_y) for valid_x, valid_y in valid_data_loader]\n",
        "    # return the mean of all the accuracies\n",
        "    return torch.stack(all_accs).mean()\n",
        "\n",
        "  \n",
        "  def train_model(self, epochs, data_loader, valid_data_loader):\n",
        "    # loop through the number of epochs provided\n",
        "    for i in range(epochs):\n",
        "      self.train_epoch(data_loader)\n",
        "      print(self.validate_epoch(valid_data_loader))\n",
        "  \n",
        "  \n",
        "  \n",
        "  "
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lvk-KQ8zIFIv"
      },
      "source": [
        "# create an optimizer class to add to the linear model\n",
        "\n",
        "from typing import List\n",
        "class SGDOptimizer:\n",
        "\n",
        "  def __init__(self, params: List[tensor], lr: int):\n",
        "    self.params = params\n",
        "    self.lr = lr\n",
        "  \n",
        "  # make a function to step wrt each parameter in the model\n",
        "  def step(self):\n",
        "    # print('step')\n",
        "    for param in self.params:\n",
        "      # take the data of the param since it requires_grad_\n",
        "      param.data -= param.grad * self.lr\n",
        "  \n",
        "  # reset the gradients for new steps\n",
        "  def zero_grad(self):\n",
        "    # print('zeroing')\n",
        "    for param in self.params:\n",
        "      param.grad = None"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xGl6GcknEKI-",
        "outputId": "7b01a626-9c44-48ec-dc3a-7c998c6d0bc4"
      },
      "source": [
        "\n",
        "basic_linear_1 = LinearModel((28*28, 1), 0.1)\n",
        "w, b = basic_linear_1.get_params()\n",
        "w.shape, b.shape"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(torch.Size([784, 1]), torch.Size([1]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "szz_wBOpER0A",
        "outputId": "154998e4-7c1e-4641-b89d-b610dcea4809"
      },
      "source": [
        "# manually make preds, evaluate loss and take gradients\n",
        "basic_linear_1.apply_linear(train_x[:4])\n",
        "basic_linear_1.linear_loss(train_x, train_y[: 4])\n",
        "basic_linear_1.loss.backward()\n",
        "\n",
        "# get the gradient of loss\n",
        "basic_linear_1.weights.grad.mean(), basic_linear_1.bias.grad\n"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor(-5.9512e-08), tensor([-4.1723e-07]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QDr4PZsIOOUm",
        "outputId": "bbcf60d6-c85b-4b38-add2-1ab3261162e0"
      },
      "source": [
        "basic_linear_1.weights.grad.zero_()\n",
        "basic_linear_1.bias.grad.zero_()"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([0.])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "swbpVSg9Gcyd",
        "outputId": "0b45a384-bc68-4331-dacb-aa87edd71b15"
      },
      "source": [
        "# create data loaders for iteration in batches by the linear model\n",
        "to_data_loader = lambda x, y, batch_size: DataLoader(list(zip(x, y)), batch_size=batch_size)\n",
        "\n",
        "train_dl = to_data_loader(train_x, train_y, 256)\n",
        "valid_dl = to_data_loader(valid_x, valid_y, 256)\n",
        "\n",
        "first(train_dl)[0].shape, first(train_dl)[1].shape"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(torch.Size([256, 784]), torch.Size([256, 1]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LgRY_X1RQZQD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fffcc8e1-9a4b-4fda-e13c-9f369cfb829e"
      },
      "source": [
        "# create a new linear model and train for 20 epochs\n",
        "\n",
        "new_linear = LinearModel((28*28, 1), 1)\n",
        "new_linear.weights.shape"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([784, 1])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TyyLHZUbiHo2",
        "outputId": "acf3101c-b474-46b2-becd-2dc550db9b39"
      },
      "source": [
        "new_linear.train_model(20, train_dl, valid_dl)"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor(0.7070)\n",
            "tensor(0.8540)\n",
            "tensor(0.9082)\n",
            "tensor(0.9292)\n",
            "tensor(0.9365)\n",
            "tensor(0.9443)\n",
            "tensor(0.9482)\n",
            "tensor(0.9541)\n",
            "tensor(0.9589)\n",
            "tensor(0.9609)\n",
            "tensor(0.9623)\n",
            "tensor(0.9633)\n",
            "tensor(0.9662)\n",
            "tensor(0.9667)\n",
            "tensor(0.9682)\n",
            "tensor(0.9682)\n",
            "tensor(0.9687)\n",
            "tensor(0.9687)\n",
            "tensor(0.9692)\n",
            "tensor(0.9702)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y0TTelm8iL_B"
      },
      "source": [
        "# create the abstraction of a complete learner\n",
        "\n",
        "class SimpleLearner:\n",
        "\n",
        "  def __init__(self, dls, model, opt_func, loss_func, metrics):\n",
        "    self.dls = dls\n",
        "    self.model = model\n",
        "    self.optim = opt_func(list(model.parameters()), 1)\n",
        "    self.loss = loss_func\n",
        "    self.metrics = metrics\n",
        "\n",
        "  def train_epoch(self):\n",
        "    # print('train')\n",
        "    # take an x and y in the dls iterator \n",
        "    for train_x, train_y in self.dls.train:\n",
        "      preds = self.model(train_x)\n",
        "      loss = self.loss(preds, train_y)\n",
        "      loss.backward()\n",
        "      self.optim.step()\n",
        "      self.optim.zero_grad()\n",
        "    \n",
        "  def epoch_acc(self):\n",
        "    # get a list of all of the epoch accuracies\n",
        "    all_accs = [self.metrics(self.model(valid_x), valid_y) for valid_x, valid_y in self.dls.valid]\n",
        "    # print(all_accs)\n",
        "    stacked_accs = torch.stack(all_accs)\n",
        "    # print(stacked_accs)\n",
        "    return stacked_accs.mean()\n",
        "\n",
        "  def fit(self, epochs):\n",
        "    for i in range(epochs):\n",
        "      # train a single epochs\n",
        "      self.train_epoch()\n",
        "\n",
        "      # output the epoch accuracy to the screen\n",
        "      print(self.epoch_acc())"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3leZi5BWragE",
        "outputId": "caf6e2a9-c190-4e1b-f349-b01beaad239c"
      },
      "source": [
        "\n",
        "# create a linear model using pytorch to test the learner\n",
        "torch_linear = nn.Linear(28*28, 1)\n",
        "x, y = torch_linear.parameters()\n",
        "x.shape, y.shape"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(torch.Size([1, 784]), torch.Size([1]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LmM5kXt9r5dE"
      },
      "source": [
        "dls = DataLoaders(train_dl, valid_dl)"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d8GRv6XHsPot"
      },
      "source": [
        "def batch_acc(preds_x, batch_y):\n",
        "  preds = preds_x.sigmoid()\n",
        "\n",
        "  # get a tensor of corret preds\n",
        "  correct = (preds > 0.5) == batch_y\n",
        "  return correct.float().mean()\n"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ERIEfsnCrnrV"
      },
      "source": [
        "\n",
        "learner = SimpleLearner(dls, torch_linear, SGDOptimizer, loss, batch_acc)  "
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0ut70BeBswc7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fb86622b-0af1-44ab-f27f-f24c4935bedf"
      },
      "source": [
        "learner.fit(20)"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor(0.4932)\n",
            "tensor(0.8691)\n",
            "tensor(0.8237)\n",
            "tensor(0.9092)\n",
            "tensor(0.9326)\n",
            "tensor(0.9463)\n",
            "tensor(0.9546)\n",
            "tensor(0.9629)\n",
            "tensor(0.9653)\n",
            "tensor(0.9668)\n",
            "tensor(0.9687)\n",
            "tensor(0.9707)\n",
            "tensor(0.9726)\n",
            "tensor(0.9751)\n",
            "tensor(0.9761)\n",
            "tensor(0.9770)\n",
            "tensor(0.9775)\n",
            "tensor(0.9780)\n",
            "tensor(0.9785)\n",
            "tensor(0.9785)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FdavTHdAszPW"
      },
      "source": [
        "# create a model with two layers and an activation between\n",
        "class SimpleModel:\n",
        "  def __init__(self, sizes: List[Tuple]):\n",
        "    # print(sizes)\n",
        "    self.weights = []\n",
        "    self.biases = []\n",
        "    for size in sizes:\n",
        "      # print(size)\n",
        "      self.weights.append((torch.randn(size)).requires_grad_())\n",
        "      # have the same number of biases as the number of layer outputs\n",
        "      self.biases.append((torch.randn(size[1])).requires_grad_())\n",
        "    \n",
        "  def __call__(self, train_x, weights=[], biases=[]):\n",
        "    # on initial call, populate weights and biases\n",
        "    weights = self.weights\n",
        "    biases = self.biases\n",
        "    return self.model(train_x, self.weights, self.biases)\n",
        "\n",
        "  \n",
        "  def model(self, inputs, weights, biases):\n",
        "    # recursive call function - BC length of list is zero\n",
        "    if len(weights) == 0:\n",
        "      return inputs\n",
        "    \n",
        "    # apply the next layer using matrix mult\n",
        "    res = inputs@weights[0] + biases[0]\n",
        "    \n",
        "    # base case has not been reached - check if there is more than one layer remaining\n",
        "    if len(weights) > 1:\n",
        "      # apply activation ReLu\n",
        "      res = res.max(tensor(0.0))\n",
        "    \n",
        "    # recursively call the function with the new activations\n",
        "    return self.model(res, weights[1:], biases[1:])\n",
        "  \n",
        "\n",
        "  # create a function to return the parameters as a tuple\n",
        "  def parameters(self):\n",
        "    # print(self.weights)\n",
        "    # return_l = []\n",
        "    # for i in range(len(self.weights)):\n",
        "    #   return_l += self.weights[i]\n",
        "    #   return_l += self.biases[i]\n",
        "    return (self.weights + self.biases)\n",
        "\n",
        "    "
      ],
      "execution_count": 58,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aAB8JZzLsgWx",
        "outputId": "98adf5cf-bc19-416d-fd0b-1f9b01ce5ce7"
      },
      "source": [
        "manual_simple_net = SimpleModel([(28*28, 30), (30, 1)])\n",
        "\n",
        "params = manual_simple_net.parameters()\n",
        "\n",
        "for param in params:\n",
        "  print(param.shape)"
      ],
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([784, 30])\n",
            "torch.Size([30, 1])\n",
            "torch.Size([30])\n",
            "torch.Size([1])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8QTVdxcky_Si",
        "outputId": "9bf695e7-89f7-4e16-f7c8-1eeb645cf704"
      },
      "source": [
        "learner_manual_model = SimpleLearner(dls, manual_simple_net, SGDOptimizer, loss, batch_acc)\n",
        "learner_manual_model.fit(20)"
      ],
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor(0.5415)\n",
            "tensor(0.7437)\n",
            "tensor(0.8506)\n",
            "tensor(0.8965)\n",
            "tensor(0.9258)\n",
            "tensor(0.9370)\n",
            "tensor(0.9448)\n",
            "tensor(0.9512)\n",
            "tensor(0.9551)\n",
            "tensor(0.9604)\n",
            "tensor(0.9614)\n",
            "tensor(0.9629)\n",
            "tensor(0.9663)\n",
            "tensor(0.9697)\n",
            "tensor(0.9721)\n",
            "tensor(0.9746)\n",
            "tensor(0.9761)\n",
            "tensor(0.9761)\n",
            "tensor(0.9765)\n",
            "tensor(0.9765)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O92gQfzHmWfm"
      },
      "source": [
        "# use pytoch to create a simple NN with 2 layers and a ReLU activation layer\n",
        "\n",
        "torch_simple_net = nn.Sequential(\n",
        "    nn.Linear(28*28, 30),\n",
        "    nn.ReLU(),\n",
        "    nn.Linear(30, 1)\n",
        ")"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c0UeXM1DmoQu",
        "outputId": "e7dacd3f-eec3-4146-b742-69c49cdb351e"
      },
      "source": [
        "all_params = torch_simple_net.parameters()\n",
        "\n",
        "for param in all_params:\n",
        "  print(param.shape)"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([30, 784])\n",
            "torch.Size([30])\n",
            "torch.Size([1, 30])\n",
            "torch.Size([1])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EgDhEVjSmqdn",
        "outputId": "fe1a3bd6-5e70-49c6-98e1-29407e5c737d"
      },
      "source": [
        "learner_torch_model = SimpleLearner(dls, torch_simple_net, SGDOptimizer, loss, batch_acc)\n",
        "\n",
        "learner_torch_model.fit(20)"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor(0.5068)\n",
            "tensor(0.7422)\n",
            "tensor(0.8955)\n",
            "tensor(0.9302)\n",
            "tensor(0.9424)\n",
            "tensor(0.9521)\n",
            "tensor(0.9565)\n",
            "tensor(0.9604)\n",
            "tensor(0.9634)\n",
            "tensor(0.9634)\n",
            "tensor(0.9648)\n",
            "tensor(0.9673)\n",
            "tensor(0.9692)\n",
            "tensor(0.9707)\n",
            "tensor(0.9712)\n",
            "tensor(0.9731)\n",
            "tensor(0.9741)\n",
            "tensor(0.9746)\n",
            "tensor(0.9761)\n",
            "tensor(0.9756)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "35xuhDU8nDXo"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}