U47768_CH05_LSTM(감성분석_Modeling).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/submouse9903/uos-deepLearning/blob/main/U47768_CH05_LSTM(%EA%B0%90%EC%84%B1%EB%B6%84%EC%84%9D_Modeling).ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Dc-SB-sESGLv"
      },
      "source": [
        "# RNN을 이용한 텍스트 분류(Text Classification)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tcnRfutbR2i1"
      },
      "source": [
        "## 1) 데이터 전처리 \n",
        "\n",
        "### - 데이터\n",
        ": 이 실습에서 사용하는 데이터는 https://github.com/lih0905/korean-pytorch-sentiment-analysis 에 있는 영화 평점 데이터를 활용하였다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qAbns-a8SD26",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "aad33c1c-fc2f-4f42-f572-75a65680b064"
      },
      "source": [
        "!pip install konlpy\n",
        "import pandas as pd \n",
        "import numpy as np\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torchtext\n",
        "from konlpy.tag import Okt\n",
        "import collections\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting konlpy\n",
            "  Downloading konlpy-0.6.0-py2.py3-none-any.whl (19.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m19.4/19.4 MB\u001b[0m \u001b[31m32.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting JPype1>=0.7.0 (from konlpy)\n",
            "  Downloading JPype1-1.4.1-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (465 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m465.3/465.3 kB\u001b[0m \u001b[31m20.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: lxml>=4.1.0 in /usr/local/lib/python3.10/dist-packages (from konlpy) (4.9.2)\n",
            "Requirement already satisfied: numpy>=1.6 in /usr/local/lib/python3.10/dist-packages (from konlpy) (1.22.4)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from JPype1>=0.7.0->konlpy) (23.1)\n",
            "Installing collected packages: JPype1, konlpy\n",
            "Successfully installed JPype1-1.4.1 konlpy-0.6.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "okt=Okt()  "
      ],
      "metadata": {
        "id": "xdbgaQr5VaFo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_data = pd.read_csv(\"http://ranking.uos.ac.kr/class/RB/rating_train.csv\")\n",
        "test_data = pd.read_csv(\"http://ranking.uos.ac.kr/class/RB/rating_test.csv\")"
      ],
      "metadata": {
        "id": "55Mjl5JuWFZb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_data.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "Z90Ap068Wr64",
        "outputId": "e4bcb2f8-7ff5-45c5-d63d-1a6074a64c44"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "         id                                               text  label\n",
              "0   9976970                                아 더빙.. 진짜 짜증나네요 목소리      0\n",
              "1   3819312                  흠...포스터보고 초딩영화줄....오버연기조차 가볍지 않구나      1\n",
              "2  10265843                                  너무재밓었다그래서보는것을추천한다      0\n",
              "3   9045019                      교도소 이야기구먼 ..솔직히 재미는 없다..평점 조정      0\n",
              "4   6483659  사이몬페그의 익살스런 연기가 돋보였던 영화!스파이더맨에서 늙어보이기만 했던 커스틴 ...      1"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e4420cc5-52c5-4c2c-8a11-5bcd05756c51\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>9976970</td>\n",
              "      <td>아 더빙.. 진짜 짜증나네요 목소리</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3819312</td>\n",
              "      <td>흠...포스터보고 초딩영화줄....오버연기조차 가볍지 않구나</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>10265843</td>\n",
              "      <td>너무재밓었다그래서보는것을추천한다</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>9045019</td>\n",
              "      <td>교도소 이야기구먼 ..솔직히 재미는 없다..평점 조정</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>6483659</td>\n",
              "      <td>사이몬페그의 익살스런 연기가 돋보였던 영화!스파이더맨에서 늙어보이기만 했던 커스틴 ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e4420cc5-52c5-4c2c-8a11-5bcd05756c51')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e4420cc5-52c5-4c2c-8a11-5bcd05756c51 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e4420cc5-52c5-4c2c-8a11-5bcd05756c51');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_data = train_data[0:1000]"
      ],
      "metadata": {
        "id": "blBVHvMkRHz3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-I8ruCnLSgpy",
        "outputId": "0890929b-5831-4d7b-d5b5-12e2233b5808"
      },
      "source": [
        "from tqdm import tqdm \n",
        "morph_counter = collections.defaultdict(int)\n",
        "text_line = train_data[\"text\"]\n",
        "for text in tqdm(text_line):\n",
        "  for m in okt.morphs(text):\n",
        "    morph_counter[m] += 1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1000/1000 [00:17<00:00, 57.73it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "morph_to_id = {'[PAD]': 0, '[UNK]': 1}\n",
        "for w, cnt in tqdm(morph_counter.items()):\n",
        "    morph_to_id[w] = len(morph_to_id)\n",
        "print(len(morph_to_id))"
      ],
      "metadata": {
        "id": "x-ZG4r5hhLmA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "195b7855-9472-4c61-95e1-e381386fc669"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 4770/4770 [00:00<00:00, 557091.58it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "4772\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "id_to_morph = {}\n",
        "for w, idx in tqdm(morph_to_id.items()):\n",
        "  id_to_morph[idx] = w\n",
        "print(len(id_to_morph))  "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8tPAD-ee9rwn",
        "outputId": "6b5eba0f-ca11-4b8a-9dcb-720586631750"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 4772/4772 [00:00<00:00, 813461.44it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "4772\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# pre-padding\n",
        "def textEncoding(text_list, max_length, morph_to_id):\n",
        "  m =  max_length-len(text_list)\n",
        "  if m>=0:\n",
        "    text_list_id = [0 if i<m else morph_to_id.get(text_list[i-m],1) for i in range(max_length)]\n",
        "  # 입력 text 가 최대 길이보다 긴 경우\n",
        "  else:\n",
        "    text_list_id = [morph_to_id.get(text_list[i],1) for i in range(max_length)]\n",
        "  return text_list_id"
      ],
      "metadata": {
        "id": "_Ub8VnecS4y4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "max_length = 10\n",
        "X = []\n",
        "for text in tqdm(train_data[\"text\"]):\n",
        "  text_list = okt.morphs(text)\n",
        "  X.append(textEncoding(text_list, max_length , morph_to_id))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CPPwBHQYTlwq",
        "outputId": "a504daf9-6ae0-4b77-fe11-c22422f6dadc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1000/1000 [00:04<00:00, 223.95it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from torch.utils.data import TensorDataset\n",
        "from torch.utils.data import DataLoader\n",
        "X_t = torch.tensor(X).long()\n",
        "Y_t = torch.tensor(train_data[\"label\"]).long()\n",
        "tr_data = TensorDataset(X_t, Y_t)\n",
        "train_dataloader = DataLoader(tr_data, batch_size=5, shuffle=True)"
      ],
      "metadata": {
        "id": "ANAM-LmmGV2j"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Y_t"
      ],
      "metadata": {
        "id": "0apS2QDb-cn2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "embedding = torch.nn.Embedding(len(morph_to_id), 40, padding_idx = 0)"
      ],
      "metadata": {
        "id": "2V5AQR_UXMat"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for x, y in train_dataloader:\n",
        "  z = embedding(x)\n",
        "  print(\"shape of embedding vector:\", z.shape)\n",
        "  break"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m-bSJY1hZDqF",
        "outputId": "bf71b0e8-aeaf-4236-eb6b-0cc6df91de7e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "shape of embedding vector: torch.Size([5, 10, 40])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q0plcGas-lhv",
        "outputId": "a3817e8b-4b86-40e6-f791-7bb34f0645d7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([0, 1, 0, 1, 0])"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 2) 모델링\n",
        "\n",
        "### - 모형"
      ],
      "metadata": {
        "id": "tYa_MpevcYRH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "torch.manual_seed(1)\n",
        "if device == 'cuda':\n",
        "    torch.cuda.manual_seed_all(1)"
      ],
      "metadata": {
        "id": "KBcZfR_9q-zG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Parameters:\n",
        "- input_size – The number of expected features in the input x\n",
        "\n",
        "- hidden_size – The number of features in the hidden state h\n",
        "\n",
        "- num_layers – Number of recurrent layers. E.g., setting num_layers=2 would mean stacking two LSTMs together to form a stacked LSTM, with the second LSTM taking in outputs of the first LSTM and computing the final results. Default: 1\n",
        "\n",
        "- bias – If False, then the layer does not use bias weights b_ih and b_hh. Default: True\n",
        "\n",
        "- batch_first – If True, then the input and output tensors are provided as (batch, seq, feature) instead of (seq, batch, feature). Note that this does not apply to hidden or cell states. See the Inputs/Outputs sections below for details. Default: False\n",
        "\n",
        "- dropout – If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, with dropout probability equal to dropout. Default: 0\n",
        "\n",
        "- bidirectional – If True, becomes a bidirectional LSTM. Default: False\n",
        "\n",
        "- proj_size – If > 0, will use LSTM with projections of corresponding size. Default: 0\n"
      ],
      "metadata": {
        "id": "TKfelvkRSZP7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# 10차원 벡터가 입력\n",
        "# 20차원 데이터가 hidden vector \n",
        "# 2개 층으로 구성\n",
        "rnn = nn.LSTM(input_size = 10, hidden_size = 20, num_layers = 2, \n",
        "              batch_first = True)"
      ],
      "metadata": {
        "id": "YIQDvVDDSMRj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Input:\n",
        "- Batch_first = False 인 경우에 (seq, batch, feature) 로 input data x 를 입력한다. (Default 값으로 지정되어 있다)\n",
        "- Batch_first = True 인 경우에 (batch, seq, feature) 로 input data x 를 입력한다. \n",
        "- 출력값 역시 Batch_first 옵션을 따른다."
      ],
      "metadata": {
        "id": "FzE8leb7TxFW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# sample 3개, 길이 5, feature 20\n",
        "input = torch.randn(3, 5, 10)"
      ],
      "metadata": {
        "id": "wVfFJQLsTt7N"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### h0 \n",
        "- Batch_first = True 와 상관없이 (num_layers*D, batch, hidden_size) 로 h0 를 입력한다.  (만약 bidirectional 이면 D = 2, 아니면 D = 1)\n",
        "- 위의 예제에서는 2개 층, 3개 샘플,  20차원이므로 (2,3,20)를 입력한다.\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "A61db5jmUNsG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "h0 = torch.randn(2, 3, 20)\n",
        "c0 = torch.randn(2, 3, 20)\n",
        "output, (hn, cn) = rnn(input, (h0, c0))"
      ],
      "metadata": {
        "id": "NTi7N2kbUM1v"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(hn.shape)\n",
        "print(cn.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qZIc-1QEWcsx",
        "outputId": "54c030c1-c5ec-464b-ce69-46e9002e72c5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([2, 3, 20])\n",
            "torch.Size([2, 3, 20])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "output.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lYMbAjxtWruY",
        "outputId": "f3c7497e-5325-4866-ea03-1d454d5a8b2f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([3, 5, 20])"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the RNN model\n",
        "class myLSTM(nn.Module):\n",
        "    def __init__(self, size_vocab, input_size, hidden_size, num_layers):\n",
        "        super(myLSTM, self).__init__()\n",
        "        self.hidden_size = hidden_size\n",
        "        self.num_layers = num_layers\n",
        "        self.embedding = nn.Embedding(size_vocab, hidden_size, padding_idx = 0)\n",
        "        self.rnn = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)\n",
        "        self.fc = nn.Sequential(nn.Linear(hidden_size, 2),\n",
        "                                nn.Softmax(dim=1))\n",
        "        \n",
        "    def forward(self, x):\n",
        "        h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(device)\n",
        "        c0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(device)\n",
        "        x = self.embedding(x).to(device)\n",
        "        out, _ = self.rnn(x, (h0,c0))\n",
        "        out = out[:, -1, :]\n",
        "        out = self.fc(out)\n",
        "        return out"
      ],
      "metadata": {
        "id": "3hl-xbv2nctW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for x, y in train_dataloader:\n",
        "  z = embedding(x)\n",
        "  print(\"shape of embedding vector:\", z.shape)\n",
        "  break"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0VrZgYxYqQg9",
        "outputId": "2aa058e5-9191-4a0f-e0c0-160a8737b17e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "shape of embedding vector: torch.Size([5, 10, 40])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "size_vocab = len(morph_to_id)\n",
        "input_size = max_length\n",
        "hidden_size = 10\n",
        "num_layer = 1"
      ],
      "metadata": {
        "id": "7h145iLZ3KLJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = myLSTM(size_vocab, input_size, hidden_size, num_layer)\n",
        "for x, y in train_dataloader:\n",
        "  z = model(x)\n",
        "  print(\"shape of embedding vector:\", z.shape)\n",
        "  break"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vCV4LsUy2tC8",
        "outputId": "efdc81d6-bda3-482f-ccb3-338ce44ebfee"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "shape of embedding vector: torch.Size([5, 2])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "learning_rate = 0.005\n",
        "model = myLSTM(size_vocab, input_size, hidden_size, num_layer).to(device)\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
        "optimizer.zero_grad()\n"
      ],
      "metadata": {
        "id": "puiYXvRU4s8r"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "num_epochs = 10\n",
        "for epoch in range(num_epochs):\n",
        "  for inputs, labels in tqdm(train_dataloader):\n",
        "    input = inputs.to(device)\n",
        "    label = labels.to(device)\n",
        "    output = model(input)\n",
        "    loss = criterion(output, label)\n",
        "    optimizer.zero_grad()\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "  print(loss)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tzMW8jAF8_io",
        "outputId": "3ee91b3c-2f65-41b1-bf22-d86f52152c42"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 200/200 [00:00<00:00, 333.62it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor(0.7103, grad_fn=<NllLossBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 200/200 [00:00<00:00, 307.49it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor(0.7604, grad_fn=<NllLossBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 200/200 [00:00<00:00, 451.20it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor(0.4497, grad_fn=<NllLossBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 200/200 [00:00<00:00, 444.78it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor(0.3370, grad_fn=<NllLossBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 200/200 [00:00<00:00, 463.02it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor(0.3139, grad_fn=<NllLossBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 200/200 [00:00<00:00, 448.46it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor(0.3202, grad_fn=<NllLossBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 200/200 [00:00<00:00, 459.76it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor(0.3187, grad_fn=<NllLossBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 200/200 [00:00<00:00, 443.13it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor(0.5143, grad_fn=<NllLossBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 200/200 [00:00<00:00, 455.69it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor(0.3210, grad_fn=<NllLossBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 200/200 [00:00<00:00, 327.30it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor(0.3146, grad_fn=<NllLossBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model(input)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DtUusnfGXe5s",
        "outputId": "6fc398bb-7ef8-437b-e241-65d6bb67ed4d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[5.9117e-03, 9.9409e-01],\n",
              "        [3.0324e-05, 9.9997e-01],\n",
              "        [9.9983e-01, 1.7464e-04],\n",
              "        [6.0048e-03, 9.9400e-01],\n",
              "        [1.7696e-05, 9.9998e-01]], grad_fn=<SoftmaxBackward0>)"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "-grtmjnIXleX"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}