12_nlp_deep_dive_redo.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "12_nlp_deep_dive_redo.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u8JMhhooxh60",
        "outputId": "d24e76fc-fb2e-444b-bfea-4ae3c2f68d85"
      },
      "source": [
        "!pip install -Uqq fastbook\n",
        "import fastbook\n",
        "fastbook.setup_book()\n",
        "from fastbook import *"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[K     |████████████████████████████████| 720 kB 5.4 MB/s \n",
            "\u001b[K     |████████████████████████████████| 46 kB 2.3 MB/s \n",
            "\u001b[K     |████████████████████████████████| 186 kB 34.3 MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.2 MB 30.5 MB/s \n",
            "\u001b[K     |████████████████████████████████| 56 kB 2.4 MB/s \n",
            "\u001b[K     |████████████████████████████████| 51 kB 211 kB/s \n",
            "\u001b[?25hMounted at /content/gdrive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "id": "Raz-vxl2y6mK",
        "outputId": "88fafe1b-21ff-467e-89b1-c6d2beff61d9"
      },
      "source": [
        "from fastai.text.all import *\n",
        "path = untar_data(URLs.HUMAN_NUMBERS)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "                background: #F44336;\n",
              "            }\n",
              "        </style>\n",
              "      <progress value='32768' class='' max='30252' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      108.32% [32768/30252 00:00<00:00]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OyOYCQmNzK7X"
      },
      "source": [
        "Path.BASE_PATH = path"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XelDpuxCzNeZ",
        "outputId": "0ac80c06-70ab-4d69-a2b0-84cc63d0c4f2"
      },
      "source": [
        "lines = L()\n",
        "with open(path/'train.txt') as f: lines += L(*f.readlines())\n",
        "with open(path/'valid.txt') as f: lines += L(*f.readlines())\n",
        "lines"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(#9998) ['one \\n','two \\n','three \\n','four \\n','five \\n','six \\n','seven \\n','eight \\n','nine \\n','ten \\n'...]"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jnksChHlz5OB"
      },
      "source": [
        "### Data Preprocessing\n",
        "- Preprocess the dataset for the language model\n",
        "- Manually tokenize and numericalize the data\n",
        "- Create batches for the model to read"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "id": "n_SJZ1P2zQYL",
        "outputId": "92dc2ee9-6208-4917-dc69-34147e8e662d"
      },
      "source": [
        "text = \" . \".join(line.strip() for line in lines)\n",
        "text[:50]"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'one . two . three . four . five . six . seven . ei'"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sq_fEat_07gj",
        "outputId": "48752f29-70b0-48dd-e5ae-25153e4a486f"
      },
      "source": [
        "# get the tokens by splitting on spaces\n",
        "tokens = text.split()\n",
        "tokens[:10]"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['one', '.', 'two', '.', 'three', '.', 'four', '.', 'five', '.']"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "t_J9UXVd1FPy",
        "outputId": "aab5e88a-0f57-46e7-d501-331c361add4a"
      },
      "source": [
        "vocab = list(set(tokens))\n",
        "vocab[:10]"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['twenty',\n",
              " 'three',\n",
              " '.',\n",
              " 'fourteen',\n",
              " 'seventeen',\n",
              " 'fifty',\n",
              " 'nine',\n",
              " 'one',\n",
              " 'thirty',\n",
              " 'forty']"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0NlHi48J1QfZ",
        "outputId": "3274b465-d126-4dec-d4d5-1cc2ca27bf60"
      },
      "source": [
        "word2index = {word: i for i, word in enumerate(vocab)}\n",
        "word2index['two']"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "21"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5_U2eQPm1nGo",
        "outputId": "7baa79b5-3d83-4d1a-e18c-4d34a43e3f5a"
      },
      "source": [
        "num_tokens = [word2index[word] for word in tokens]\n",
        "num_tokens[:10]"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[7, 2, 21, 2, 1, 2, 28, 2, 19, 2]"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p-8zFVpc1tkz",
        "outputId": "67781cb9-4c18-4a6b-d546-3d0ff0b00111"
      },
      "source": [
        "len_tks = 6\n",
        "\n",
        "for i in range(0,len_tks-4,3):\n",
        "  print(i)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FobGrNvb2kHU",
        "outputId": "81f9a736-df57-4117-d9c7-85d4e6bd3184"
      },
      "source": [
        "print(len(tokens))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "63095\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-1hM-xlw5OA2",
        "outputId": "21af18f8-e7b1-46d8-e5fc-598c70ac2a42"
      },
      "source": [
        "# want sequences of consecutive characters\n",
        "seq_len = 16\n",
        "seqs = [(torch.Tensor(num_tokens[i: i+seq_len]).long(), torch.Tensor(num_tokens[i+1:i+seq_len+1]).long()) for i in range(0,len(num_tokens)-seq_len-1, seq_len)]\n",
        "seqs[:2]                                                                                         "
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(tensor([ 7,  2, 21,  2,  1,  2, 28,  2, 19,  2, 14,  2, 16,  2, 18,  2]),\n",
              "  tensor([ 2, 21,  2,  1,  2, 28,  2, 19,  2, 14,  2, 16,  2, 18,  2,  6])),\n",
              " (tensor([ 6,  2, 29,  2, 22,  2, 26,  2, 10,  2,  3,  2, 20,  2, 13,  2]),\n",
              "  tensor([ 2, 29,  2, 22,  2, 26,  2, 10,  2,  3,  2, 20,  2, 13,  2,  4]))]"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "svHE4W8p2ox0"
      },
      "source": [
        "# want data in order\n",
        "# (0, m, 2m, ..., )\n",
        "\n",
        "def group_chunks(dset, bs):\n",
        "  m = len(dset) // bs\n",
        "  new_dset = []\n",
        "\n",
        "  for i in range(m):\n",
        "    new_dset += [dset[i + m*j] for j in range(bs)]\n",
        "  return new_dset"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EFxhtjY45DhL"
      },
      "source": [
        "bs = 64\n",
        "cut = int(len(seqs) * 0.8)\n",
        "dls = DataLoaders.from_dsets(\n",
        "    group_chunks(seqs[:cut], bs), \n",
        "    group_chunks(seqs[cut:], bs), \n",
        "    bs=bs, drop_last=True, shuffle=False)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "igAdPdXT17FV",
        "outputId": "b08fa837-8cef-4d5e-e5bd-91a84cb70064"
      },
      "source": [
        "x, y = dls.one_batch()\n",
        "x.shape, y.shape"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(torch.Size([64, 16]), torch.Size([64, 16]))"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qnAg_mkB7ba7"
      },
      "source": [
        "### Experiment with RNNs\n",
        "- Write RNNs from scratch"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hv3xunCD5NQo"
      },
      "source": [
        "class LMModel1(Module):\n",
        "  def __init__(self, vocab_sz, n_hidden):\n",
        "    self.i_h = nn.Embedding(vocab_sz, n_hidden)\n",
        "    self.h_h = nn.Linear(n_hidden, n_hidden)\n",
        "    self.h_o = nn.Linear(n_hidden, vocab_sz)\n",
        "    self.h = 0\n",
        "    self.relu = nn.ReLU()\n",
        "  \n",
        "  def forward(self, x):\n",
        "    # print(x.shape)\n",
        "    outs = []\n",
        "    for i in range(x.shape[1]):\n",
        "      # add the hidden state to the input to hidden state\n",
        "      self.h = self.h + self.i_h(x[:, i])\n",
        "      # compute new hidden state\n",
        "      self.h = self.relu(self.h_h(self.h))\n",
        "      # make a prediction for new character\n",
        "      outs.append(self.h_o(self.h))\n",
        "\n",
        "      \n",
        "    self.h = self.h.detach()\n",
        "    outs = torch.stack(outs, dim=1)\n",
        "    # print(outs.shape)\n",
        "\n",
        "    return outs\n",
        "  \n",
        "  \n",
        "  def reset(self):\n",
        "    self.h = 0"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qcBySyFk_ybR"
      },
      "source": [
        "def flat_cross_entropy(inp, targ):\n",
        "  # flatten the target from (bs, seq) -> (bs*seq)\n",
        "  # accomodate by flattening inputs (bs, seq, vocab_sz) -> (bs*seq, vocab_sz)\n",
        "  return F.cross_entropy(inp.view(bs*seq_len, -1), targ.view(-1))"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NZDxJYIN9655"
      },
      "source": [
        "def loss_func(inp, targ):\n",
        "    return F.cross_entropy(inp.view(-1, len(vocab)), targ.view(-1))"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PTqEtzKs12KJ",
        "outputId": "adaa3390-7bd9-45f0-961c-4df00eea90de"
      },
      "source": [
        "print(len(vocab))"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "30\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 302
        },
        "id": "n9jANdP57an3",
        "outputId": "a175ce11-23ee-440a-99c1-38b7599e171d"
      },
      "source": [
        "learn = Learner(dls, LMModel1(len(vocab), 64), loss_func=flat_cross_entropy,\n",
        "                metrics=accuracy, cbs=ModelResetter)\n",
        "learn.lr_find()"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "SuggestedLRs(valley=0.0010000000474974513)"
            ]
          },
          "metadata": {},
          "execution_count": 20
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEMCAYAAADJQLEhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU9b3/8ddnkgkJEBIMAUFE9n0RReteBQWrdWnr8mtt1V6rbfVqba239nZTu9z2eqvWrS7V2lqtpbihVWtVFFcUEKiICrKDSNhCgEBm+fz+mEkIMQlJyMls7+fjMY/MnPOdcz5nCPPJ93w3c3dERCR3hVIdgIiIpJYSgYhIjlMiEBHJcUoEIiI5TolARCTHKRGIiOS4wBKBmRWa2VtmNt/MFprZdU2UO8fM3kuWeSioeEREpHEW1DgCMzOgi7tvM7Mw8CrwHXd/s16ZIcBUYKK7bzaznu6+PpCARESkUflBHdgTGWZb8mU4+WiYdS4Gbnf3zcn37DUJ9OjRw/v379+OkYqIZL85c+ZscPfyxvYFlggAzCwPmAMMJvGFP6tBkaHJcq8BecC17v5sI8e5BLgEoF+/fsyePTvIsEVEso6ZrWhqX6CNxe4ec/eDgb7A4WY2ukGRfGAIcDzwZeAeMytt5Dh3u/sEd59QXt5oQhMRkTbqkF5D7r4FmAGc3GDXamC6u0fcfRnwIYnEICIiHSTIXkPltX/dm1kRcBLwfoNij5OoDWBmPUjcKloaVEwiIvJpQbYR9Ab+lGwnCAFT3f0pM7semO3u04F/ApPN7D0gBlzt7htbe6JIJMLq1avZuXNne8afUQoLC+nbty/hcDjVoYhIhgms+2hQJkyY4A0bi5ctW0ZxcTFlZWUkeq3mFndn48aNVFVVMWDAgFSHIyJpyMzmuPuExvZlxcjinTt35mwSADAzysrKcrpGJCJtlxWJAMjZJFAr169fJNs9/94nLFlfFcixsyYRZJKuXbsCsHz5ckaPbtijVkTk0y59cC6PzF0TyLFzMxEsmAo3jYZrSxM/F0xNdUQiIk1yd2piccKhYGr+uZcIFkyFJ6+AylWAJ34+ecU+JYNrrrmG22+/ve71tddeyy9+8QsmTZrEIYccwpgxY3jiiSeaPUYsFuPqq6/msMMOY+zYsdx1110AnH/++Tz++ON15c4777y9HktEskssnujUE84L5is79xLBC9dDpHrPbZHqxPY2Ovfcc5k6dXcimTp1KhdccAGPPfYYc+fOZcaMGVx11VU010Pr3nvvpaSkhLfffpu3336be+65h2XLlnHRRRdx//33A1BZWcnrr7/Oqaee2uZYRSTzRGKJ7478gBJBoHMNpaXK1a3b3gLjx49n/fr1rF27loqKCrp3787+++/Pd7/7XWbOnEkoFGLNmjV88skn7L///o0e47nnnmPBggVMmzYtEU5lJYsXL2by5MlceumlVFRU8Mgjj/ClL32J/Pzc+2cTyWWReByAcF4wt4Zy7xulpG/ytlAj2/fB2WefzbRp01i3bh3nnnsuDz74IBUVFcyZM4dwOEz//v2b7d7p7tx6661MmTLlU/vOP/98/vKXv/Dwww/zxz/+cZ/iFJHME4nWJgLdGmofk34K4aI9t4WLEtv3wbnnnsvDDz/MtGnTOPvss6msrKRnz56Ew2FmzJjBihVNTvwHwJQpU/j9739PJBIB4MMPP2T79u0AXHjhhdx8880AjBw5cp/iFJHME43X3hpSjaB9jD0n8fOF6xO3g0r6JpJA7fY2GjVqFFVVVRxwwAH07t2b8847j9NOO40xY8YwYcIEhg8f3uz7v/GNb7B8+XIOOeQQ3J3y8vK6RuJevXoxYsQIzjzzzH2KUUQyU03ANYKsmGJi0aJFjBgxIkURBW/Hjh2MGTOGuXPnUlJS0mS5bP8cRHLVsg3bOeH/XuKmc8fxhfFtu42d9VNMZLPnn3+eESNGcPnllzebBEQke0ViwdYIcu/WUIY58cQT99q+ICLZrTYR5IfUWCwikpOiyXEEBfkaWdysTGvraG+5fv0i2Uw1ghYoLCxk48aNOftlWLseQWFhYapDEZEA1I4sVhtBM/r27cvq1aupqKhIdSgpU7tCmYhkn92NxRpH0KRwOKyVuUQka0XjGlksIpLTaqLBjixWIhARSXO1NYIC1QhERHJTXa8hJQIRkdy0u9eQbg2JiOSkoKeYUCIQEUlz0YDHESgRiIikud1tBLo1JCKSk2rbCNRrSEQkR+2ea0g1AhGRnBRNJoI8JQIRkdxUE3MK8kKYKRGIiOSkaCweWEMxKBGIiKS9aNwD6zoKSgQiImmvJhYPbFQxKBGIiKS9aCyemTUCMys0s7fMbL6ZLTSz65op+yUzczObEFQ8IiKZKhLzQNsIglyYZhcw0d23mVkYeNXMnnH3N+sXMrNi4DvArABjERHJWJFMrRF4wrbky3Dy0diiwj8HfgPsDCoWEZFMFonFCQe0cD0E3EZgZnlmNg9YD/zL3Wc12H8IcKC7/2Mvx7nEzGab2excXpdYRHJTNOaE8zO0sdjdY+5+MNAXONzMRtfuM7MQcCNwVQuOc7e7T3D3CeXl5cEFLCKShmpicfIztUZQy923ADOAk+ttLgZGAy+Z2XLgCGC6GoxFRPYUTY4sDkqQvYbKzaw0+bwIOAl4v3a/u1e6ew937+/u/YE3gdPdfXZQMYmIZKJIBo8s7g3MMLMFwNsk2gieMrPrzez0AM8rIpJVIgGPLA6s+6i7LwDGN7L9p02UPz6oWEREMlkkqpHFIiI5LRrP0HEEIiLSPhIjizPw1lA6c3fWbd1JRdUuQmaEzDCDrdURNm2vYcP2GjZvr2FHTYydkcQjMfufEc4LkR8KUROLsWNXjB01MaojMeLuyWM3f+64O5FYnJqYUxONE487oZARssSiEyEz8kKWfA6xuBOJOdF4nGjMcYeYe935aiuLZkY87nX73BPHy0teW+01GomfDWOKxZ24154vzq5onJponJpYnFDy/aHkG6PxOLG4E407RuI8Zol4a49vyZgsGWRtufrXWFsGEmuxFuSFCOeFKMgPEc5LlMkPhQjVlU0cJ3EtVhdX7fWR/JlnEEpee911J6+59ph5ISM/zyjMz6MwnEdhOESXgny6FYUp7RympChMcWE+xYVhijvlEwpoQRCRlogEPOlcziSCD9ZV8eg7q3lv7VYWrt3Kpu01e31PQV6IwnCIooI88syIxJ1oLE4k5nTKT2zvUpBPYTixYET9L8CmGBDOC9G5IPGllxeCuFPvSxxiyS/aeBzyQyEKw5Ysm/xyCzU4h4Pju5NIcl/t8eLx3cnBSfys/3Yj+QUdSnyJFuQnvowL8vISg1g8kSzinkh0tV/SeSHD6+1LnKP2POxxPkjsj8Y9ca1x36NMLJ5IjLVJqDbRVCeTML67fO21xL3+de25PRrzPfYnPybiyePW/jvWJFd+2puunfKTiSGRHEqKwpQWhSntXEBp5zDdO4fp3qWA7p0Tjx5dC9ivS0Ggf8VJ7gh6ZHHOJIJVm3Zw36vLGNqrmBNH9GRUnxIOKC2q9wXnFBeGKetaQFmXArp3KQj0npykh3jc2RWNUx2JsX1XlMrqCFurI4mfOyNU7YyydWeUrdURtu2KUpXc9snWnXywrorK5PbGmEH3zgX0LO7EAaVF9Ek+DtyviIP260K/ss6UFIU7+IolE0UzeNK5tHLc0HIWXncyBfn6cpfdQiGjqCCPooI89utSwIFtOEZNNM6W6ho2b4+weUfituKGbbuo2Jb4+UnlTtZW7mT2is1UVkf2eG9p5zCDyrsyqLwLg3t2ZUivYkb17kbPboXtc4GSFWoCnnQuZxKBEoAEpSA/RM/iQnoW7/3Lu2pnhFWbqlm5aQcrN21n2YYdLK3YxovvVzB19uq6cj26dmJUn26M71fKhIP2Y3y/Urp0ypn/rtJANOZqIxDJFsWFYUb2CTOyT7dP7duyo4YP1lWxMNmOtXBtJb97YXFdw//I3t04clAZRw4q4/D+++1ODAumwgvXQ+VqKOkLk34KY8/p4CuTIAXdfVSJQCRNlHYu4DMDy/jMwLK6bVt3Rnhn5RZmL9/ErGWbuP+15dw9cyn5IeOQft25pPtsJi7+JaFodeINlavgySsSz5UMsoK7q/uoSC7rVhjms0PL+ezQxKy71TUx5qzYzGsfbeDlDyoYvvBmQqHqPd8UqU7UEJQIskI0nuj2VqBbQyICUFSQxzFDenDMkB784OTh+LUbGy3nlaup2hmhW6F6JWW6SLKLc5A1ArWgimQwK+nb6PY18TKO+NUL/PSJd1myflujZSQzRGKJGoGmmBCRxk36KYSL9twWLiI+8Sd8bnRvHn5rFSfe+DIX/vEtFqzekpoYZZ/U1gg06ZyING7sOXDaLVByIGCJn6fdQr/jL+S354zj9R9O5KqThjJv1RZOv+01vvnAbD78pCrVUUsrRDugRqA2ApFMN/acJhuGe3TtxOWThnDh0f2599Vl/OGVZTz33ky+fHg/rvnccLUhZIC6NoIA57tSjUAkBxQXhrnyxKG88l8n8PWjBvDwWys56caXeW7hulSHJntRmwiCHBSrRCCSQ7p3KeCnp43ksUuPpnvnAi55YA6XPTiXLTv2PgmjpEZtY3HGL14vIull3IGlPHn5MVw9ZRjPvbeOU295VY3JaUqNxSISmHBeiMtOGMzfv3UUAGf9/g0eeHMFvrdFNaRD7U4EqhGISEAOPrCUpy4/hqMGl/GTx9/lqr/Pr/vykdSrHVmsRCAigerepYD7LjiMK08cwqNz13Dpg3PZFY2lOiwhsXA9EOh6BEoEIgIk1ma48sShXHf6KP713idc/Oc5VNcoGaRaRDUCEeloFxzVn998aQyvLK7g6/e/1eQKbNIxamsEaiwWkQ517mH9uPncg3l7+WYue3AusbgakFMlGldjsYikyBkHH8DPzxjNyx9W8D9PL0p1ODmrpm6KCU1DLSIp8JXP9OPDT6r4w6vLGLZ/MWdPaMuqzrIvonVTTKhGICIp8uNTR3DM4B786LF3mbNiU6rDyTl1k85pigkRSZX8vBC3fWU8fUoL+eYDc/hk685Uh5RTamoHlGnSORFJpdLOBfzhgglU7Yzyo8fe1ejjDhTVyGIRSReDexbz/cnDeH7RJ0yfvzbV4eSMuknn1H1URNLBfxwzgPH9SvnZ9IVUVO1KdTg5IaLuoyKSTvJCxg1njWVHTYyfPK5bRB0hEs3gkcVmVmhmb5nZfDNbaGbXNVLme2b2npktMLMXzOygoOIRkfYxuGcx3z1xKM8uXMc//v1xqsPJetF4nJAlknBQgqwR7AImuvs44GDgZDM7okGZd4AJ7j4WmAb8b4DxiEg7ufjYAYzrW8K10xdqCoqA1cTi5AdYG4AAE4EnbEu+DCcf3qDMDHffkXz5JtA3qHhEpP3k54W4/ozRbNhWwx0zlqQ6nKwWjTkFmZoIAMwsz8zmAeuBf7n7rGaKXwQ808RxLjGz2WY2u6KiIohQRaSVxh1YyhfGH8AfXl3G6s079v4GaZNILB5ojyEIOBG4e8zdDybxl/7hZja6sXJm9lVgAnBDE8e5290nuPuE8vLy4AIWkVa5esowQga/efaDVIeStSIxD7ShGDqo15C7bwFmACc33GdmJwI/Ak53d/VHE8kgfUqLuOTYgTw5fy1zVmxOdThZKRKLBzqqGILtNVRuZqXJ50XAScD7DcqMB+4ikQTWBxWLiATnm58dRM/iTvziH++pO2kAorF4oPMMQbA1gt7ADDNbALxNoo3gKTO73sxOT5a5AegK/N3M5pnZ9ADjEZEAdOmUz/enDOOdlVt4coG6k7a3SMzJD7hGENg01O6+ABjfyPaf1nt+YlDnF5GOc9YhffnT68v5zTPvM3lkLwrDeakOKWtEYvHsaCMQkewWChk/PnUka7ZUc++ry1IdTlZRIhCRjHHkoDImj+zFHTOWsL5KU1W3l2jcA12dDJQIRKQd/fCUEdTE4tz0rw9THUrWiGTyyGIRyT0DenTh/CP787e3V7Ho462pDicrRDJ9ZLGI5J4rJg6hW1FY3UnbSTTTRxaLSO4p6RzmyklDeG3JRl58X8OD9lVNzANduB6UCEQkAOcdcRADe3ThV08vIpJcalHaJhqLU5CvGoGIZJhwXohrPjecjyq28/BbK1MdTkaLxOKqEYhIZjppZC8+M2A/bnp+MVt3RlIdTsbKmknnRCT3mCUGmW3aXsMdMz5KdTgZKzGgLA1uDZlZFzMLJZ8PNbPTzSwcaGQikvHG9C3hi+MP4L7XlrFqk9YsaIvEgLL0qBHMBArN7ADgOeBrwP1BBSUi2eP7U4ZhwP89pzUL2iISTZ/uo5ZcUvKLwB3ufjYwKriwRCRb9Ckt4j+OGcD0+WtZsn7b3t8ge4jE42kzoMzM7EjgPOAfyW2aXlBEWuQbxwygU36IO17S+satFYl52tQIrgR+CDzm7gvNbCCJFcdERPaqrGsnvnL4QTwxb63aClohHndi6dJG4O4vu/vp7v6bZKPxBne/ItDIRCSrXHLcQPLM+P3L6kHUUpF4YjBeWiQCM3vIzLqZWRfgXeA9M7s60MhEJKvsX1LIWRP6Mm32atZVaprqlojGEnM1pUX3UWCku28FzgSeAQaQ6DkkItJi3/7sIGLu3D1zaapDyQi103Oky8jicHLcwJnAdHePAJpWUERa5cD9OnPGwX146K0VbNy2K9XhpL1IbY0gTRavvwtYDnQBZprZQYAmGxeRVrv0+MHsisa1pGUL1NYIwgEvXt/SxuJb3P0Adz/FE1YAJwQamYhkpcE9u3LK6N488MYKKqs1B1FzdrcRpEGNwMxKzOxGM5udfPyWRO1ARKTVLj1hEFW7ovz59eWpDiWt1fYaSpdxBPcBVcA5ycdW4I9BBSUi2W1UnxImDe/Jva8tY/uuaKrDSVu1t4bSZWTxIHf/mbsvTT6uAwYGGZiIZLfLJg5my44ID83SegVNqb01lC6L11eb2TG1L8zsaKA6mJBEJBcc0q87Rw8u4+5XlrIzEkt1OGmppraxOE1uDX0LuN3MlpvZcuA24JuBRSUiOeGyEwZTUbWLv89elepQ0lJaNRa7+3x3HweMBca6+3hgYqCRiUjWO3JgGYce1J07X15KTVRrGze0e0BZetQIAHD3rckRxgDfCyAeEckhZsZ/ThzMmi3VPPbO6lSHk3bqxhGkyYCyxgSbokQkJxw/tJyxfUu4bcaSui8+SagbWZwmU0w0RlNMiMg+MzOumDiEVZuqeeydNakOJ61E62oEKbw1ZGZVZra1kUcV0CfQyEQkZ0wa0ZPRB3Tj9hlL6r78ZHevoZROOufuxe7erZFHsbvnBxqZiOSM2lrBio07eHze2lSHkzZqew2ly4AyEZFAnTSyFyN7d+O2FxerVpBU12soTcYRtJqZFZrZW2Y238wWmtl1jZTpZGZ/M7MlZjbLzPoHFY+IpDcz44pJQ1i+cQdPLlCtACAST6NxBG20C5iYHH9wMHCymR3RoMxFwGZ3HwzcBPwmwHhEJM1NHtmL4fsXc+sLS4jF1R8lEk2vkcWtlpyuelvyZTj5aPgvewbwp+TzacAkM1O3VJEcFQoZ35k0hKUbtvPkfNUKoum0ZnFbmVmemc0D1gP/cvdZDYocAKwCcPcoUAmUNXKcS2qnwK6oqAgyZBFJsSmj9mf4/sXc8sLinK8VROomncvQGgGAu8fc/WCgL3C4mY1u43HudvcJ7j6hvLy8fYMUkbSiWsFuu1coy+AaQS133wLMAE5usGsNcCCAmeUDJcDGjohJRNKXagUJ0ZiTFzJC6TTXUGuYWbmZlSafFwEnAe83KDYduCD5/CzgRXfP3X91EQFUK6gVicUDbyiGYGsEvYEZZrYAeJtEG8FTZna9mZ2eLHMvUGZmS0hMYndNgPGISAZRrSDRRhD0bSGAwEYHu/sCYHwj239a7/lO4OygYhCRzFVbK/j2g3OZPn8NXxjfN9UhdbhILB74zKOgkcUiksZqawU3P784J2cmjcbjga9FAEoEIpLGQiHjqsnDWLFxB9Pm5N56BTVRD3wMASgRiEiaO3FET8YdWMotLyzOubWNo/HMbywWEdlnZsbVk4fxceVO/vrWylSH06EisTj5qhGIiMDRg8s4YuB+3D5jCTtqoqkOp8NEYro1JCICJGsFU4axYVsN97++PNXhdJhsGEcgItJuDj1oP04YVs5dLy9l685IqsPpEFHVCERE9nTV5GFUVkf402vLUx1Kh6iJqfuoiMgeRh9QwqThPbn3tWVs25X9bQXRWJwCDSgTEdnT5ZOGsGVHhL+8uSLVoQQuEnPVCEREGjr4wFKOHdKDe2Yupbomu8cVJBqLVSMQEfmUKyYNYeP2Gh6cld21AiUCEZEmHNZ/P44YuB93z1ya1aONo3FX91ERkaZcMXEI66t2MXX2qlSHEphIVCOLRUSadOSgMg49qDt3vvQRNdHsnJk0Etc4AhGRJpkZV0wawtrKnTwyNztnJo1qZLGISPOOG9KDcX1LuOOlJVm5XoHmGhIR2Qsz4/KJQ1i1qZon5mXf2saJ2UdVIxARadakET0Z2bsbt89YknVrG0dicQpUIxARaV6irWAwyzZs56kF2VMriMWduEN+Byxer0QgIhlv8sj9GdarmNteXEI8S2oFtW0e4XzdGhIR2atQyLhs4mAWr9/GM++uS3U47SKaTGhh1QhERFrm1DG9GdyzKzc9/2FWtBVEkmMj1H1URKSF8kLGVScNZcn6bTyaBeMKIvFEItDIYhGRVjh59P6M7VvCzc8vZlc0s+cgisSSt4ZUIxARabnatY3XbKnmoVkrUx3OPonWNharRiAi0jrHDO7BkQPLuO3FJWzP4FXMansN6daQiEgrmRlXnzyMjdtruO/VZakOp81qbw0V6NaQiEjrHdKvOyeN7MXdM5eyeXtNqsNpk7oagbqPioi0zdVThrGtJsrvX/4o1aG0SV1jsRavFxFpm6G9ivni+L7c//py1m6pTnU4rVY3sliL14uItN13TxoCDr97fnGqQ2m1aDbUCMzsQDObYWbvmdlCM/tOI2VKzOxJM5ufLPP1oOIRkdzTt3tnvnrEQfx9ziqWrK9KdTitsruNILNrBFHgKncfCRwBXGZmIxuUuQx4z93HAccDvzWzggBjEpEcc9kJg+hckM8N//wg1aG0SiQbxhG4+8fuPjf5vApYBBzQsBhQbGYGdAU2kUggIiLtoqxrJy4+diD/XPgJ76zcnOpwWqxu0rlMTgT1mVl/YDwwq8Gu24ARwFrg38B33P1T682Z2SVmNtvMZldUVAQcrYhkm28cO4AeXQv49TPv454ZE9LtrhFk9q0hAMysK/AIcKW7b22wewowD+gDHAzcZmbdGh7D3e929wnuPqG8vDzokEUky3TplM93Jg1h1rJNPPfeJ6kOp0V2zzWU4TUCMwuTSAIPuvujjRT5OvCoJywBlgHDg4xJRHLTlw/vx5CeXfnV04syYkK6rGgjSN73vxdY5O43NlFsJTApWb4XMAxYGlRMIpK78vNC/OTzI1mxcQd/en15qsPZq2jdXEOZfWvoaOBrwEQzm5d8nGJm3zKzbyXL/Bw4ysz+DbwA/MDdNwQYk4jksOOGljNxeE9ufWEJG7ftSnU4zarpwFtD+UEd2N1fBZpNZe6+FpgcVAwiIg399ykjOPnmmdz4rw/55RfGpDqcJkWzqbFYRCSdDO7Zla8ecRB/fWsl769r2H8lfWRFG4GISLq68sQhdCsKc/2T76Vtd9LaXkOZPrJYRCQtlXYu4KqThvL6Rxt59t11qQ6nUZFYnHCekeh3EywlAhHJSV8+vB/D9y/mF/9YRHVN+nUnjca9Q9YiACUCEclR+Xkhrj19FGu2VHPXzPRbs6AmGu+QrqOgRCAiOeyIgWV8fmxvfv/SR6zevCPV4ewhGo9T0AENxaBEICI57r9PGYEZ/OrpRakOZQ+RqKtGICLSEfqUFnHZ8YN5+t/reOmD9akOp04kFlcbgYhIR7n4uIEMKu/Cjx57l2270mMm/EXrqjiorHOHnEuJQERyXmE4j/89axxrK6v5zTPvpzocNm+vYdHHWzlyYFmHnE+JQEQEOPSg7nz9qAE88OYKZi3dmNJYZi1LnP/IQUoEIiId6vtThtJvv8784JEFKR1b8MZHGykK5zG2b2mHnE+JQEQkqXNBPr/+4hiWb9zBTc9/mLI43li6kQn9u1OQr8ZiEZEOd9TgHnz58AO599VlLPq44yel27BtFx9+sq3DbguBEoGIyKf84OThlBSF+cnj7xKPd+ykdG8m2yc6qqEYlAhERD6ltHMB15w8nNkrNvPI3NUdeu43PtpI1075jDmgpMPOqUQgItKIsw7tyyH9Svn1M+9TuSPSYed9Y+lGDuvfnfwOml4ClAhERBoVChk/P3M0m3fUcMNzHTO24JOtO1lasb1D2wdAiUBEpEmj+pRw/pH9eXDWShas3hL4+Xa3D/QI/Fz1KRGIiDTje5OH0qNrJ378+LvEAm44fuOjjXQrzGdkn26BnqchJQIRkWZ0Kwzz41NHsGB1JQ+9tTLQc72xdCOHDygjrwOWp6xPiUBEZC9OH9eHowaV8b/Pvk9F1a5AzrF2SzUrNu7o8PYBUCIQEdkrM+P6M0azMxLjf54JZt2CR5PdVI8b0rHtA6BEICLSIoN7duWbxw3i0blr6hp120vljgh3zVzKiSN6MaRXcbseuyWUCEREWuiyEwbTt3sRP378XWqi8XY77l0zP2LbrihXTR7absdsDSUCEZEWKirI4/ozRrFk/TZueWFxuxyzomoXf3xtOaeN7cOI3h3bW6iWEoGISCtMHN6Lsw7tyx0vLWHuys37fLzbZyyhJhbnuyelpjYASgQiIq32s9NG0rukiO/9bR47atq+tOWaLdU8NGslZx/alwE9urRjhK2jRCAi0krFhWF+e844Vmzawa+ebnsvoluTt5eumDSkvUJrEyUCEZE2OGJgGd84ZgB/eXMlL32wvtXvn/lhBX+bvYqvHnEQfUqLAoiw5ZQIRETa6KrJwxjaqytX/m0ec1ZsavH71myp5jsPv8PQnnFtCywAAAj0SURBVMV8f0rq2gZqKRGIiLRRYTiPP5x/GKVFYb5yzyyeW7iubl80FueROau5/K/v8NqSDXXbd0VjXPrgXKIx586vHUrngvxUhL6HwCIwswOBPwO9AAfudvffNVLueOBmIAxscPfPBhWTiEh761fWmUe+fRT/8afZfOsvc7jujNEUd8rnlhcWs3TDdgrDIZ6cv5Ypo3rxo1NGcvcrHzF/1Rbu/OqhKW0grs/cg5lNz8x6A73dfa6ZFQNzgDPd/b16ZUqB14GT3X2lmfV092Zvtk2YMMFnz54dSMwiIm21oybKfz70Di++n/gKG75/Md89aSifHVrOfa8t47YXlxCJxYnEnG8eN5AfnjKiQ+MzsznuPqHRfUElgkaCeAK4zd3/VW/bpUAfd/9xS4+jRCAi6Soai3PPK8s4qKwzJ4/an1C9WUQ/2bqT3z73ATsjcW48Z1yHrkAGaZAIzKw/MBMY7e5b622vvSU0CigGfufuf27uWEoEIiKt11wiCLyVwsy6Ao8AV9ZPAvXOfygwCSgC3jCzN939wwbHuAS4BKBfv35BhywiklMCrZuYWZhEEnjQ3R9tpMhq4J/uvt3dN5CoNYxrWMjd73b3Ce4+oby8PMiQRURyTmCJwMwMuBdY5O43NlHsCeAYM8s3s87AZ4BgJvsWEZFGBXlr6Gjga8C/zWxectt/A/0A3P1Od19kZs8CC4A48Ad3fzfAmEREpIHAEoG7vwrsdeFNd78BuCGoOEREpHkaWSwikuOUCEREcpwSgYhIjuuwkcXtxcwqgBVACVCZ3Ly357U/ewC7Z39qufrHbM3+htube6249x7X3va3Je7Gtinuve/f27amrqG94m6vz7rhtmz63a7/vAQodffG+9+7e0Y+SExi16Ln9X7O3tdztWZ/w+3NvVbcqYm7iW2Key/797atqWtor7jb67NuLu5M/91u6nNv7JHJt4aebMXz+tv29Vyt2d9we3OvFXfT52vp/rbE3dS1tEUuxb23bU1dQ3vF3V6fdcNt2fS7Xf95s+fNuFtD+8LMZnsTc22kM8XdsRR3x8rEuDMx5uZkco2gLe5OdQBtpLg7luLuWJkYdybG3KScqhGIiMin5VqNQEREGlAiEBHJcUoEIiI5TolARCTHKREkmdmxZnanmf3BzF5PdTwtZWYhM/ulmd1qZhekOp6WMrPjzeyV5Gd+fKrjaQ0z62Jms83s86mOpSXMbETyc55mZt9OdTwtZWZnmtk9ZvY3M5uc6nhayswGmtm9ZjYt1bG0VFYkAjO7z8zWm9m7DbafbGYfmNkSM7umuWO4+yvu/i3gKeBPQcZbL759jhs4A+gLREis+Ba4dorbgW1AIZkVN8APgKnBRLmndvrdXpT83T6HxDohgWunuB9394uBbwHnBhlvvfjaI+6l7n5RsJG2s7YMk063B3AccAjwbr1tecBHwECgAJgPjATGkPiyr//oWe99U4HiTIkbuAb4ZvK90zIo7lDyfb1ILGWaKXGfBPw/4ELg85kQc/I9pwPPAF/JlM+63vt+CxySgXF3yP/H9ngEvnh9R3D3mWbWv8Hmw4El7r4UwMweBs5w9/8BGq3Sm1k/oNLdqwIMt057xG1mq4Ga5MtYcNHu1l6fd9JmoFMQcTbUTp/38UAXEl8E1Wb2tLvH0znm5HGmA9PN7B/AQ0HFW+987fFZG/Br4Bl3nxtsxAnt/LudMbIiETThAGBVvderSayJ3JyLgD8GFlHLtDbuR4FbzexYYGaQge1Fq+I2sy8CU4BS4LZgQ2tWq+J29x8BmNmFwIYgk0AzWvtZHw98kUTCfTrQyJrX2t/ty4ETgRIzG+zudwYZXDNa+3mXAb8ExpvZD5MJI61lcyJoNXf/WapjaC1330EigWUUd3+URBLLSO5+f6pjaCl3fwl4KcVhtJq73wLckuo4WsvdN5Jo18gYWdFY3IQ1wIH1XvdNbkt3irtjZWLcmRgzKO60lc2J4G1giJkNMLMCEg1801McU0so7o6ViXFnYsyguNNXqlur2+MB/BX4mN1dKC9Kbj8F+JBEi/+PUh2n4lbcuRCz4s68h2YfFRHJcdl8a0hERFpAiUBEJMcpEYiI5DglAhGRHKdEICKS45QIRERynBKBZAUz29bB52uXNSuS6zJUmtk8M3vfzP6vBe8508xGtsf5RUCJQKRRZtbsPFzuflQ7nu4Vdz8YGA983sz2tmbAmSRmPxVpF0oEkrXMbJCZPWtmcyyxGtrw5PbTzGyWmb1jZs+bWa/k9mvN7AEzew14IPn6PjN7ycyWmtkV9Y69Lfnz+OT+acm/6B9MTp+MmZ2S3DbHzG4xs6eai9fdq4F5JGa7xMwuNrO3zWy+mT1iZp3N7CgSawvckKxFDGrqOkVaSolAstndwOXufijwfeCO5PZXgSPcfTzwMPBf9d4zEjjR3b+cfD2cxHTZhwM/M7NwI+cZD1yZfO9A4GgzKwTuAj6XPH/53oI1s+7AEHZPJ/6oux/m7uOARSSmO3idxDw3V7v7we7+UTPXKdIimoZaspKZdQWOAv6e/AMddi+A0xf4m5n1JrHi1LJ6b52e/Mu81j/cfRewy8zWk1hRreHSmm+5++rkeecB/Uksw7nU3WuP/VfgkibCPdbM5pNIAje7+7rk9tFm9gsSazZ0Bf7ZyusUaRElAslWIWBL8t57Q7cCN7r79OSiLdfW27e9Qdld9Z7HaPz/TEvKNOcVd/+8mQ0A3jSzqe4+D7gfONPd5ycXwjm+kfc2d50iLaJbQ5KV3H0rsMzMzobEsodmNi65u4Td88lfEFAIHwAD6y17uNfF15O1h18DP0huKgY+Tt6OOq9e0arkvr1dp0iLKBFItuhsZqvrPb5H4svzouRtl4XAGcmy15K4lTIH2BBEMMnbS5cCzybPUwVUtuCtdwLHJRPIT4BZwGvA+/XKPAxcnWzsHkTT1ynSIpqGWiQgZtbV3bclexHdDix295tSHZdIQ6oRiATn4mTj8UISt6PuSnE8Io1SjUBEJMepRiAikuOUCEREcpwSgYhIjlMiEBHJcUoEIiI57v8DgmieEIOR5h4AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 669
        },
        "id": "I0gXSKxH9gxc",
        "outputId": "fb5c5f3c-5d62-4bd0-b06e-306740c381ec"
      },
      "source": [
        "learn.fit_one_cycle(20, 0.0005)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>3.323441</td>\n",
              "      <td>3.309674</td>\n",
              "      <td>0.036621</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>3.209437</td>\n",
              "      <td>3.100973</td>\n",
              "      <td>0.291016</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>2.792349</td>\n",
              "      <td>2.227321</td>\n",
              "      <td>0.416911</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>2.113846</td>\n",
              "      <td>1.895303</td>\n",
              "      <td>0.473551</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.754839</td>\n",
              "      <td>1.870028</td>\n",
              "      <td>0.474609</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>1.569274</td>\n",
              "      <td>1.816618</td>\n",
              "      <td>0.474691</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>1.452861</td>\n",
              "      <td>1.807349</td>\n",
              "      <td>0.468669</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>1.379259</td>\n",
              "      <td>1.766390</td>\n",
              "      <td>0.480876</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>1.328319</td>\n",
              "      <td>1.754932</td>\n",
              "      <td>0.499430</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>1.293603</td>\n",
              "      <td>1.732118</td>\n",
              "      <td>0.490641</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>1.265303</td>\n",
              "      <td>1.758426</td>\n",
              "      <td>0.517497</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>1.242635</td>\n",
              "      <td>1.716512</td>\n",
              "      <td>0.522624</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>1.216252</td>\n",
              "      <td>1.716262</td>\n",
              "      <td>0.522868</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>1.197175</td>\n",
              "      <td>1.722088</td>\n",
              "      <td>0.534261</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>1.182512</td>\n",
              "      <td>1.727066</td>\n",
              "      <td>0.525228</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>1.175634</td>\n",
              "      <td>1.724297</td>\n",
              "      <td>0.536947</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>1.165996</td>\n",
              "      <td>1.705571</td>\n",
              "      <td>0.526367</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>1.157043</td>\n",
              "      <td>1.712597</td>\n",
              "      <td>0.529867</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>1.152178</td>\n",
              "      <td>1.714581</td>\n",
              "      <td>0.528971</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>1.149653</td>\n",
              "      <td>1.714755</td>\n",
              "      <td>0.531576</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "831VNhNX_onc"
      },
      "source": [
        "### Multilayer RNN\n",
        "- Experiment with multilayer RNNs\n",
        "- Have a different hidden state in effect making the model deeper (more params to update)\n",
        "- Make the multilayer RNN from scratch\n",
        "- Make the multilayer RNN using pytorch"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-_knlhfh-ZAE"
      },
      "source": [
        "class LMModel2(Module):\n",
        "\n",
        "  def __init__(self, vocab_sz, n_hidden, n_layers):\n",
        "    self.n_layers = n_layers\n",
        "    self.n_hidden = n_hidden\n",
        "    self.relu = nn.ReLU()\n",
        "    # must create hidden state for each layer in the network\n",
        "    self.h = torch.zeros(n_layers, bs, n_hidden)\n",
        "\n",
        "    self.i_h = nn.Embedding(vocab_sz, n_hidden)\n",
        "    self.rnn_layers = []\n",
        "    # create the middle rnn layers according to n_layers\n",
        "    for i in range(n_layers):\n",
        "      self.rnn_layers.append(nn.Linear(n_hidden, n_hidden))\n",
        "    self.h_o = nn.Linear(n_hidden, vocab_sz)\n",
        "\n",
        "\n",
        "  def forward(self, x):\n",
        "    outs = []\n",
        "\n",
        "    # loop through the sequence of inputs\n",
        "    for i in range(x.shape[1]):\n",
        "      # update the hidden state for the first RNN layer\n",
        "      self.h.data[0] = self.h.data[0] + self.i_h(x[:, i])\n",
        "\n",
        "      # loop through multilayers and apply them\n",
        "      for i, h_layer in enumerate(self.rnn_layers):\n",
        "        if i == 0:\n",
        "          # already dealt with input to first layer above, apply hidden layer\n",
        "          self.h.data[i] = self.relu(h_layer(self.h.data[i]))\n",
        "\n",
        "        else:\n",
        "          # pass prev activations in\n",
        "          self.h.data[i] = self.h.data[i] + self.h.data[i-1]\n",
        "          # apply new RNN layer\n",
        "          self.h.data[i] = self.relu(h_layer(self.h.data[i]))\n",
        "      \n",
        "      # at the end, go hidden to output on the last set of activations\n",
        "      outs.append(self.h_o(self.h[-1]))\n",
        "\n",
        "    # detach the gradients\n",
        "    self.h = self.h.detach()\n",
        "\n",
        "    return torch.stack(outs, dim=1)\n",
        "  \n",
        "  def reset(self):\n",
        "    # reset the hidden state on epoch or validation\n",
        "    # self.h = torch.zeros(self.n_layers, bs, self.n_hidden)\n",
        "    self.h = torch.zeros_like(self.h)"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 297
        },
        "id": "QHaAjx2NCpwZ",
        "outputId": "cbe8e8ae-3cba-445f-a68a-cb2b1d5e17e4"
      },
      "source": [
        "learn = Learner(dls, LMModel2(len(vocab), 64, 2), \n",
        "                loss_func=CrossEntropyLossFlat(), \n",
        "                metrics=accuracy, cbs=ModelResetter)\n",
        "learn.fit_one_cycle(8, 3e-3)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>3.336657</td>\n",
              "      <td>3.264450</td>\n",
              "      <td>0.029053</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>3.054815</td>\n",
              "      <td>2.894682</td>\n",
              "      <td>0.124674</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>2.864093</td>\n",
              "      <td>2.818560</td>\n",
              "      <td>0.162109</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>2.783172</td>\n",
              "      <td>2.801814</td>\n",
              "      <td>0.174235</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>2.751319</td>\n",
              "      <td>2.798034</td>\n",
              "      <td>0.171387</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>2.740172</td>\n",
              "      <td>2.797850</td>\n",
              "      <td>0.148438</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>2.736782</td>\n",
              "      <td>2.797473</td>\n",
              "      <td>0.146322</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>2.735783</td>\n",
              "      <td>2.797346</td>\n",
              "      <td>0.146159</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h8jq7dAdInG1"
      },
      "source": [
        "## Realizations from above\n",
        "- After writing the RNN code, I realized that it was not training\n",
        "- You cannot index into a variable in pytorch and set it partially when requires grad; a work around would be needed for this\n",
        "- If the indexing in pytorch did not require a work around, the previous model would have worked\n",
        "- Try implementing multilayer rnn with pytorch now"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jdzf1JElCvXI"
      },
      "source": [
        "# override the previous LMModel2 class\n",
        "class LMModel2(Module):\n",
        "  def __init__(self, vocab_sz, n_hidden, n_layers):\n",
        "    self.i_h = nn.Embedding(vocab_sz, n_hidden)\n",
        "    self.rnn = nn.RNN(n_hidden, n_hidden, n_layers, batch_first=True)\n",
        "    self.h_o = nn.Linear(n_hidden, vocab_sz)\n",
        "    self.h = torch.zeros(n_layers, bs, n_hidden)\n",
        "\n",
        "  \n",
        "  def forward(self, x):\n",
        "    inps = self.i_h(x)\n",
        "    # this does all of the for loops and indexing to end with a final outs variable\n",
        "    outs, hid = self.rnn(inps, self.h)\n",
        "    # print(hid.grad)\n",
        "    self.h = self.h.detach()\n",
        "    return self.h_o(outs)\n",
        "  \n",
        "\n",
        "  def reset(self):\n",
        "    self.h.zero_()"
      ],
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 297
        },
        "id": "o4Y5bIWCCplK",
        "outputId": "5e0ec622-0d30-4135-eff3-120f773d1dc5"
      },
      "source": [
        "learn = Learner(dls, LMModel2(len(vocab), 64, 2), \n",
        "                loss_func=CrossEntropyLossFlat(), \n",
        "                metrics=accuracy, cbs=ModelResetter)\n",
        "learn.fit_one_cycle(8, 3e-3)"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>2.773918</td>\n",
              "      <td>2.088853</td>\n",
              "      <td>0.445394</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.890882</td>\n",
              "      <td>1.825922</td>\n",
              "      <td>0.440837</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.607420</td>\n",
              "      <td>1.900379</td>\n",
              "      <td>0.330892</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.471005</td>\n",
              "      <td>1.938458</td>\n",
              "      <td>0.343913</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.390500</td>\n",
              "      <td>1.917730</td>\n",
              "      <td>0.377279</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>1.337064</td>\n",
              "      <td>2.002375</td>\n",
              "      <td>0.373779</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>1.302500</td>\n",
              "      <td>2.039661</td>\n",
              "      <td>0.373779</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>1.283795</td>\n",
              "      <td>2.055036</td>\n",
              "      <td>0.374349</td>\n",
              "      <td>00:01</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C2y5-ehZha7H",
        "outputId": "44add55d-5408-4dc2-ce34-8d28b1ca499e"
      },
      "source": [
        "learn.model"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LMModel2(\n",
              "  (i_h): Embedding(30, 64)\n",
              "  (rnn): RNN(64, 64, num_layers=2, batch_first=True)\n",
              "  (h_o): Linear(in_features=64, out_features=30, bias=True)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-D6Y2918iyCr",
        "outputId": "c8a8dee5-f923-423f-8efc-7b9a0f659723"
      },
      "source": [
        "learn.model.parameters()"
      ],
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<generator object Module.parameters at 0x7f50c55a9cd0>"
            ]
          },
          "metadata": {},
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Sp3bF04OhFVi"
      },
      "source": [
        "# run one epochs and look at gradients\n",
        "# for x, y in dls.train:\n",
        "#   preds = learn.model(x)\n",
        "\n",
        "#   loss = learn.loss(preds, y)\n",
        "#   loss.backward()\n",
        "\n",
        "#   print(learn.model.parameters().grad)\n"
      ],
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qc3Y0MaoKawG",
        "outputId": "dd3cd272-0abc-45d9-fdd3-d1977f826f26"
      },
      "source": [
        "learn.model.h_o.weight"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Parameter containing:\n",
              "tensor([[-0.1346, -0.0105, -0.0292,  ...,  0.1076,  0.1697, -0.0790],\n",
              "        [-0.1446, -0.1159, -0.0657,  ...,  0.3506,  0.0702, -0.1373],\n",
              "        [-0.0230,  0.2399, -0.0219,  ..., -0.0172, -0.1246, -0.0223],\n",
              "        ...,\n",
              "        [ 0.0767, -0.2379,  0.1511,  ...,  0.1802, -0.0354, -0.0948],\n",
              "        [ 0.0534, -0.1983,  0.1668,  ..., -0.0494, -0.1688, -0.1655],\n",
              "        [-0.1557, -0.0960, -0.1385,  ..., -0.2076,  0.0217, -0.0095]], requires_grad=True)"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ao_l2LeikB9C"
      },
      "source": [
        "### Implement LSTM \n",
        "- First implement it from scratch\n",
        "- Then, implement from scratch with matrix mult and chunks in 4\n",
        "- Finally implement using pytorch"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pka4h4N6kBOc"
      },
      "source": [
        "class LSTMCell(Module):\n",
        "  def __init__(self, x_sz, h_sz):\n",
        "    # create 4 different gates, each for the type of gate in the lstm cell\n",
        "    self.forget = nn.Linear(x_sz + h_sz, h_sz)\n",
        "    self.input = nn.Linear(x_sz + h_sz, h_sz)\n",
        "    self.cell = nn.Linear(x_sz + h_sz, h_sz)\n",
        "    self.output = nn.Linear(x_sz + h_sz, h_sz)\n",
        "  \n",
        "    self.sig = nn.Sigmoid()\n",
        "    self.tanh = nn.Tanh()\n",
        "  \n",
        "\n",
        "  def forward(self, x, state):\n",
        "    # spread the hidden state\n",
        "    h, c = state\n",
        "    # concat the inputs\n",
        "    all_inps = torch.cat([x, h], dim=1)\n",
        "\n",
        "    c = c * self.sig(self.forget(all_inps))\n",
        "\n",
        "    input_combined = self.sig(self.input(all_inps)) * self.tanh(self.cell(all_inps))\n",
        "    c = c + input_combined\n",
        "\n",
        "    h = self.sig(self.output(all_inps)) * self.tanh(c)\n",
        "\n",
        "    return h, (h, c)\n",
        "\n"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7ybuGzu-gcNr"
      },
      "source": [
        "class LSTMCellSimplified(Module):\n",
        "  def __init__(self, x_sz, h_sz):\n",
        "    self.i_h = nn.Linear(x_sz, h_sz*4)\n",
        "    self.h_h = nn.Linear(h_sz, h_sz*4)\n",
        "    self.sig = nn.Sigmoid()\n",
        "    self.tanh = nn.Tanh()\n",
        "  \n",
        "  def forward(self, x, state):\n",
        "    h, c = state\n",
        "\n",
        "    # combine the x input and hidden input into 4 gates\n",
        "    all_gates = (self.i_h(x) + self.h_h(h)).chunk(4, dim=1)\n",
        "\n",
        "    # apply functinos to layers\n",
        "    forget, input, output = map(self.sig, all_gates[:3])\n",
        "    cell = self.tanh(all_gates[3])\n",
        "\n",
        "    # perform cell computations\n",
        "    c = c * forget\n",
        "    c = c + input*cell\n",
        "    # note: you may need to flip this line to:\n",
        "    # h = output * self.tanh(c)\n",
        "    h = self.tanh(c) * output\n",
        "\n",
        "    return h, (h, c)\n"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wdXEixvCZLCw"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}