Text_Preprocessing_NLP.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyMCjqqpA7q1rj3aqyOlsaMG"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-NgLzQ-S_azI"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import seaborn as sn"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount(\"/content/drive\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rq9nqqvi_peR",
        "outputId": "59344832-2c0a-49da-fa89-dd8c1e223654"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv('/content/drive/My Drive/datasets_files/IMDB_Dataset.csv')"
      ],
      "metadata": {
        "id": "uNQw3yBr_zBS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cj-d9as5_5da",
        "outputId": "7d89b792-2b47-4867-fb0d-4695f07425da"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(50000, 2)"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "Rr4qNtaq_6RH",
        "outputId": "0c7d2663-0896-4196-b385-f29c0ac7089e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                              review sentiment\n",
              "0  One of the other reviewers has mentioned that ...  positive\n",
              "1  A wonderful little production. <br /><br />The...  positive\n",
              "2  I thought this was a wonderful way to spend ti...  positive\n",
              "3  Basically there's a family where a little boy ...  negative\n",
              "4  Petter Mattei's \"Love in the Time of Money\" is...  positive"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-32a99198-596c-4851-bee4-6714ed730cb2\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>review</th>\n",
              "      <th>sentiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>One of the other reviewers has mentioned that ...</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>A wonderful little production. &lt;br /&gt;&lt;br /&gt;The...</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>I thought this was a wonderful way to spend ti...</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Basically there's a family where a little boy ...</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Petter Mattei's \"Love in the Time of Money\" is...</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-32a99198-596c-4851-bee4-6714ed730cb2')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-32a99198-596c-4851-bee4-6714ed730cb2 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-32a99198-596c-4851-bee4-6714ed730cb2');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df[\"review\"][3]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        },
        "id": "m1yehSRYATG4",
        "outputId": "ff552427-ac08-4e81-a2d1-e90a29b5f885"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"Basically there's a family where a little boy (Jake) thinks there's a zombie in his closet & his parents are fighting all the time.<br /><br />This movie is slower than a soap opera... and suddenly, Jake decides to become Rambo and kill the zombie.<br /><br />OK, first of all when you're going to make a film you must Decide if its a thriller or a drama! As a drama the movie is watchable. Parents are divorcing & arguing like in real life. And then we have Jake with his closet which totally ruins all the film! I expected to see a BOOGEYMAN similar movie, and instead i watched a drama with some meaningless thriller spots.<br /><br />3 out of 10 just for the well playing parents & descent dialogs. As for the shots with Jake: just ignore them.\""
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df[\"review\"][3].lower()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        },
        "id": "ms6OEe2OAY88",
        "outputId": "20f97c74-42c8-4b08-a4eb-a36fcd25e965"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"basically there's a family where a little boy (jake) thinks there's a zombie in his closet & his parents are fighting all the time.<br /><br />this movie is slower than a soap opera... and suddenly, jake decides to become rambo and kill the zombie.<br /><br />ok, first of all when you're going to make a film you must decide if its a thriller or a drama! as a drama the movie is watchable. parents are divorcing & arguing like in real life. and then we have jake with his closet which totally ruins all the film! i expected to see a boogeyman similar movie, and instead i watched a drama with some meaningless thriller spots.<br /><br />3 out of 10 just for the well playing parents & descent dialogs. as for the shots with jake: just ignore them.\""
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**changing into lowercase**"
      ],
      "metadata": {
        "id": "Fu3TXSJdAuuG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df[\"review\"]=df[\"review\"].str.lower()"
      ],
      "metadata": {
        "id": "kPLU3HpzAbml"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df[\"review\"]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vt2j2GyyAp6f",
        "outputId": "616d9a46-1c38-4135-e131-ac6068904744"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0        one of the other reviewers has mentioned that ...\n",
              "1        a wonderful little production. <br /><br />the...\n",
              "2        i thought this was a wonderful way to spend ti...\n",
              "3        basically there's a family where a little boy ...\n",
              "4        petter mattei's \"love in the time of money\" is...\n",
              "                               ...                        \n",
              "49995    i thought this movie did a down right good job...\n",
              "49996    bad plot, bad dialogue, bad acting, idiotic di...\n",
              "49997    i am a catholic taught in parochial elementary...\n",
              "49998    i'm going to have to disagree with the previou...\n",
              "49999    no one expects the star trek movies to be high...\n",
              "Name: review, Length: 50000, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Removing HTML tags**"
      ],
      "metadata": {
        "id": "_aWqtvK0A5Wu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# using regex\n",
        "import re\n",
        "def remove_html_tags(text):\n",
        "  pattern = re.compile(\"<.*?>\")\n",
        "  return pattern.sub(r'',text)"
      ],
      "metadata": {
        "id": "PTu9O6fKA4LD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "text = \"<html><body><p> Movie </p><p> actor</p> <h1>Prabhas</h1></body></html>\""
      ],
      "metadata": {
        "id": "W9heFEPYBqx0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "remove_html_tags(text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "6FjqWVU_B7wT",
        "outputId": "296c6c82-7d51-4fa0-df94-81b473517d86"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "' Movie  actor Prabhas'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df[\"review\"]=df[\"review\"].apply(remove_html_tags)"
      ],
      "metadata": {
        "id": "7DxY0aO_B_dQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df[\"review\"][3]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        },
        "id": "q4UzkwdJCL1G",
        "outputId": "f3282038-38d7-48e7-a1bf-5b1da6983a82"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"basically there's a family where a little boy (jake) thinks there's a zombie in his closet & his parents are fighting all the time.this movie is slower than a soap opera... and suddenly, jake decides to become rambo and kill the zombie.ok, first of all when you're going to make a film you must decide if its a thriller or a drama! as a drama the movie is watchable. parents are divorcing & arguing like in real life. and then we have jake with his closet which totally ruins all the film! i expected to see a boogeyman similar movie, and instead i watched a drama with some meaningless thriller spots.3 out of 10 just for the well playing parents & descent dialogs. as for the shots with jake: just ignore them.\""
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Remove URL's**"
      ],
      "metadata": {
        "id": "DuDDgfDHCW1J"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "text1 = \"check out my https://dialogflow.cloud.google.com/\"\n",
        "text2 = \"check google www.google.com\""
      ],
      "metadata": {
        "id": "nQXX_XpbCTeG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def remove_url(text):\n",
        "  pattern = re.compile(r'https?://\\S+|www\\.\\S+')\n",
        "  return pattern.sub(r'',text)"
      ],
      "metadata": {
        "id": "ksyQpMDFC8ew"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "remove_url(text1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "KMLGF7pEC2P4",
        "outputId": "2911bd1c-f5fe-4366-9a18-99bd2492ae3c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'check out my '"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "remove_url(text2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "GMnIqGLhDPWF",
        "outputId": "6e038e80-6c15-46d7-e676-79b6127b6ca2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'check google '"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Removing Punctuation**"
      ],
      "metadata": {
        "id": "8IG0jVDhDSHu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import string\n",
        "string.punctuation"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "q8BXQ9x4C47N",
        "outputId": "a4424472-3acc-449a-c796-4ec1f9353d1c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "exclude = string.punctuation"
      ],
      "metadata": {
        "id": "WO5i3z6UEmqY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# it is very slow\n",
        "def remove_punc(text):\n",
        "  for char in exclude:\n",
        "    text = text.replace(char,'')\n",
        "  return text"
      ],
      "metadata": {
        "id": "mXdWZvN3EvQD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "text = \"string. with.- punctuation?\""
      ],
      "metadata": {
        "id": "ecvhf_ByE8pu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "text"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "8Cxsu1KBG4Vm",
        "outputId": "de9f3de4-f0ac-40b6-b5a2-72f12d55cffd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'string. with.- punctuation?'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import time"
      ],
      "metadata": {
        "id": "YUdDT7_DGUrl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "start = time.time()\n",
        "remove_punc(text)\n",
        "time1 = time.time()-start\n",
        "print(time1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zez63_luFFnr",
        "outputId": "44f638b1-feb4-48f4-b900-0c13c461ae69"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "9.441375732421875e-05\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(time1*500000)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NNCWoADtGOgD",
        "outputId": "8a94302f-66c5-4f40-89c3-8408eba8019b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "47.206878662109375\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# another technique\n",
        "def remove_punc1(text):\n",
        "  return text.translate(str.maketrans('','',exclude))"
      ],
      "metadata": {
        "id": "qNoQR5WVFIPR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "start = time.time()\n",
        "remove_punc1(text)\n",
        "time2 = time.time()-start\n",
        "print(time2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yku8sELvFik5",
        "outputId": "d025c8df-4fa8-4398-fd1a-27203170a4e4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "6.29425048828125e-05\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(time2*500000)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sOTwlow2G6s3",
        "outputId": "741a3487-133a-46d3-bd0a-fca08e7bb211"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "31.47125244140625\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**chat word treatment**"
      ],
      "metadata": {
        "id": "-qpgAmmcIDsW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# like gm,hlo,hi,dm,gn\n",
        "chat_words = {\n",
        "    \"LOL\": \"Laughing Out Loud\",\n",
        "    \"BRB\": \"Be Right Back\",\n",
        "    \"OMG\": \"Oh My God\",\n",
        "    \"IMO\": \"In My Opinion\",\n",
        "    \"ASAP\": \"As Soon As Possible\",\n",
        "    \"TBH\": \"To Be Honest\",\n",
        "    \"IDK\": \"I Don't Know\",\n",
        "    \"TMI\": \"Too Much Information\",\n",
        "    \"WTF\": \"What The F***\",\n",
        "    \"AFK\": \"Away From Keyboard\",\n",
        "    \"FYI\": \"For Your Information\",\n",
        "    \"IRL\": \"In Real Life\",\n",
        "    \"BTW\": \"By The Way\",\n",
        "    \"G2G\": \"Got To Go\",\n",
        "    \"ROFL\": \"Rolling On the Floor Laughing\",\n",
        "    \"NVM\": \"Never Mind\",\n",
        "    \"ICYMI\": \"In Case You Missed It\",\n",
        "    \"JK\": \"Just Kidding\",\n",
        "    \"LMAO\": \"Laughing My A** Off\",\n",
        "    \"OOTD\": \"Outfit Of The Day\",\n",
        "    \"TGIF\": \"Thank God It's Friday\",\n",
        "    \"YOLO\": \"You Only Live Once\",\n",
        "    \"SMH\": \"Shaking My Head\",\n",
        "    \"NSFW\": \"Not Safe For Work\",\n",
        "    \"ICYMI\": \"In Case You Missed It\",\n",
        "    \"HBD\": \"Happy Birthday\",\n",
        "    \"IMHO\": \"In My Humble Opinion\",\n",
        "    \"BFF\": \"Best Friends Forever\",\n",
        "    \"NBD\": \"No Big Deal\",\n",
        "    \"PPL\": \"People\",\n",
        "    \"TTYL\": \"Talk To You Later\",\n",
        "    \"IKR\": \"I Know Right\",\n",
        "    \"IRL\": \"In Real Life\",\n",
        "    \"CBA\": \"Can't Be Arsed\",\n",
        "    \"FOMO\": \"Fear Of Missing Out\",\n",
        "    \"KYS\": \"Kill Yourself\",\n",
        "    \"TY\": \"Thank You\",\n",
        "    \"WCW\": \"Woman Crush Wednesday\",\n",
        "    \"MCM\": \"Man Crush Monday\",\n",
        "    \"AFAIK\": \"As Far As I Know\",\n",
        "    \"SFW\": \"Safe For Work\",\n",
        "    \"BRUH\": \"Brother\",\n",
        "    \"GG\": \"Good Game\",\n",
        "    \"SMH\": \"Shaking My Head\",\n",
        "    \"TFW\": \"That Feeling When\",\n",
        "    \"TBF\": \"To Be Fair\",\n",
        "    \"OMW\": \"On My Way\",\n",
        "    \"MFW\": \"My Face When\"\n",
        "}\n"
      ],
      "metadata": {
        "id": "GvlxChowHPzQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def chat_conversion(text):\n",
        "  new_text = []\n",
        "  for w in text.split():\n",
        "    if w.upper() in chat_words:\n",
        "      new_text.append(chat_words[w.upper()])\n",
        "    else:\n",
        "      new_text.append(w)\n",
        "  return \" \".join(new_text)"
      ],
      "metadata": {
        "id": "q22JffeWJd_H"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "chat_conversion(\"TY ra\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "LTvt1XYyJ64Y",
        "outputId": "7144f1db-87b0-43c1-944d-8041bdf69daf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Thank You ra'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "chat_conversion(\"IMHO he is the best\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "hkYm_NXOKBfR",
        "outputId": "70e2dc98-09ad-4e81-855e-19ba9e8618fc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'In My Humble Opinion he is the best'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 61
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**spelling correction**"
      ],
      "metadata": {
        "id": "OU1YHklkKMkq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from textblob import TextBlob"
      ],
      "metadata": {
        "id": "jvGWGdylKF8C"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "incorrect_text = \"ceeritain conditionas negetive impression\"\n",
        "textblb = TextBlob(incorrect_text)\n",
        "textblb.correct().string"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "MpblFWGSKhEM",
        "outputId": "05b5cdc8-f37b-44f1-afa5-c6cda9ea3160"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'certain conditions negative impression'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**remove stopwords**"
      ],
      "metadata": {
        "id": "MN1nxAPRK5pU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# rempve stop words\n",
        "import nltk\n",
        "from nltk.corpus import stopwords"
      ],
      "metadata": {
        "id": "C21gGCQiKyqi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "nltk.download(\"stopwords\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_a-TRMvzLOUz",
        "outputId": "69bb05e0-a145-4a86-c5dd-a12db5336882"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 71
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "stopwords.words(\"english\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ygt6LxbPLXKE",
        "outputId": "bfbc7530-55ef-4d6d-fafa-60931e0a771e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['i',\n",
              " 'me',\n",
              " 'my',\n",
              " 'myself',\n",
              " 'we',\n",
              " 'our',\n",
              " 'ours',\n",
              " 'ourselves',\n",
              " 'you',\n",
              " \"you're\",\n",
              " \"you've\",\n",
              " \"you'll\",\n",
              " \"you'd\",\n",
              " 'your',\n",
              " 'yours',\n",
              " 'yourself',\n",
              " 'yourselves',\n",
              " 'he',\n",
              " 'him',\n",
              " 'his',\n",
              " 'himself',\n",
              " 'she',\n",
              " \"she's\",\n",
              " 'her',\n",
              " 'hers',\n",
              " 'herself',\n",
              " 'it',\n",
              " \"it's\",\n",
              " 'its',\n",
              " 'itself',\n",
              " 'they',\n",
              " 'them',\n",
              " 'their',\n",
              " 'theirs',\n",
              " 'themselves',\n",
              " 'what',\n",
              " 'which',\n",
              " 'who',\n",
              " 'whom',\n",
              " 'this',\n",
              " 'that',\n",
              " \"that'll\",\n",
              " 'these',\n",
              " 'those',\n",
              " 'am',\n",
              " 'is',\n",
              " 'are',\n",
              " 'was',\n",
              " 'were',\n",
              " 'be',\n",
              " 'been',\n",
              " 'being',\n",
              " 'have',\n",
              " 'has',\n",
              " 'had',\n",
              " 'having',\n",
              " 'do',\n",
              " 'does',\n",
              " 'did',\n",
              " 'doing',\n",
              " 'a',\n",
              " 'an',\n",
              " 'the',\n",
              " 'and',\n",
              " 'but',\n",
              " 'if',\n",
              " 'or',\n",
              " 'because',\n",
              " 'as',\n",
              " 'until',\n",
              " 'while',\n",
              " 'of',\n",
              " 'at',\n",
              " 'by',\n",
              " 'for',\n",
              " 'with',\n",
              " 'about',\n",
              " 'against',\n",
              " 'between',\n",
              " 'into',\n",
              " 'through',\n",
              " 'during',\n",
              " 'before',\n",
              " 'after',\n",
              " 'above',\n",
              " 'below',\n",
              " 'to',\n",
              " 'from',\n",
              " 'up',\n",
              " 'down',\n",
              " 'in',\n",
              " 'out',\n",
              " 'on',\n",
              " 'off',\n",
              " 'over',\n",
              " 'under',\n",
              " 'again',\n",
              " 'further',\n",
              " 'then',\n",
              " 'once',\n",
              " 'here',\n",
              " 'there',\n",
              " 'when',\n",
              " 'where',\n",
              " 'why',\n",
              " 'how',\n",
              " 'all',\n",
              " 'any',\n",
              " 'both',\n",
              " 'each',\n",
              " 'few',\n",
              " 'more',\n",
              " 'most',\n",
              " 'other',\n",
              " 'some',\n",
              " 'such',\n",
              " 'no',\n",
              " 'nor',\n",
              " 'not',\n",
              " 'only',\n",
              " 'own',\n",
              " 'same',\n",
              " 'so',\n",
              " 'than',\n",
              " 'too',\n",
              " 'very',\n",
              " 's',\n",
              " 't',\n",
              " 'can',\n",
              " 'will',\n",
              " 'just',\n",
              " 'don',\n",
              " \"don't\",\n",
              " 'should',\n",
              " \"should've\",\n",
              " 'now',\n",
              " 'd',\n",
              " 'll',\n",
              " 'm',\n",
              " 'o',\n",
              " 're',\n",
              " 've',\n",
              " 'y',\n",
              " 'ain',\n",
              " 'aren',\n",
              " \"aren't\",\n",
              " 'couldn',\n",
              " \"couldn't\",\n",
              " 'didn',\n",
              " \"didn't\",\n",
              " 'doesn',\n",
              " \"doesn't\",\n",
              " 'hadn',\n",
              " \"hadn't\",\n",
              " 'hasn',\n",
              " \"hasn't\",\n",
              " 'haven',\n",
              " \"haven't\",\n",
              " 'isn',\n",
              " \"isn't\",\n",
              " 'ma',\n",
              " 'mightn',\n",
              " \"mightn't\",\n",
              " 'mustn',\n",
              " \"mustn't\",\n",
              " 'needn',\n",
              " \"needn't\",\n",
              " 'shan',\n",
              " \"shan't\",\n",
              " 'shouldn',\n",
              " \"shouldn't\",\n",
              " 'wasn',\n",
              " \"wasn't\",\n",
              " 'weren',\n",
              " \"weren't\",\n",
              " 'won',\n",
              " \"won't\",\n",
              " 'wouldn',\n",
              " \"wouldn't\"]"
            ]
          },
          "metadata": {},
          "execution_count": 72
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def remove_stopwords(text):\n",
        "  new_text=[]\n",
        "\n",
        "  for word in text.split():\n",
        "    if word in stopwords.words(\"english\"):\n",
        "      new_text.append('')\n",
        "    else:\n",
        "      new_text.append(word)\n",
        "  x=new_text[:]\n",
        "  new_text.clear()\n",
        "  return \" \".join(x)"
      ],
      "metadata": {
        "id": "YVutE4WqMH5v"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "remove_stopwords(\"the sun rises in east direction,so i am very happy\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "L58fCZWPMKdn",
        "outputId": "87a05ac2-d997-4eed-d674-bf110cdabef0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "' sun rises  east direction,so    happy'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 76
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df[\"review\"]=df[\"review\"].apply(remove_stopwords)"
      ],
      "metadata": {
        "id": "-M_C3DRnN6Ax"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Handling Emoji's**"
      ],
      "metadata": {
        "id": "sA_3NYCVNXVm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "\n",
        "def remove_emojis(text):\n",
        "    emoji_pattern = re.compile(\"[\"\n",
        "                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
        "                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
        "                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
        "                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
        "                               u\"\\U00002500-\\U00002BEF\"  # chinese char\n",
        "                               u\"\\U00002702-\\U000027B0\"\n",
        "                               u\"\\U00002702-\\U000027B0\"\n",
        "                               u\"\\U000024C2-\\U0001F251\"\n",
        "                               u\"\\U0001f926-\\U0001f937\"\n",
        "                               u\"\\U00010000-\\U0010ffff\"\n",
        "                               u\"\\u2640-\\u2642\"\n",
        "                               u\"\\u2600-\\u2B55\"\n",
        "                               u\"\\u200d\"\n",
        "                               u\"\\u23cf\"\n",
        "                               u\"\\u23e9\"\n",
        "                               u\"\\u231a\"\n",
        "                               u\"\\ufe0f\"  # dingbats\n",
        "                               u\"\\u3030\"\n",
        "                               \"]+\", flags=re.UNICODE)\n",
        "    return emoji_pattern.sub(r'', text)\n"
      ],
      "metadata": {
        "id": "L8ijqaF2NLiy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "remove_emojis(\"loved the film, it was 😀 😁 😂 🤣 😃 \")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "jhL1dFlhN61_",
        "outputId": "d4237c9a-e2f7-4fc7-d0ef-0dae1570bcb7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'loved the film, it was      '"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 79
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pip install emoji --upgrade"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0rKtWIpgOl4N",
        "outputId": "fc5cc11a-a6c6-49c9-c08c-5dce91ec44e2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting emoji\n",
            "  Downloading emoji-2.2.0.tar.gz (240 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m240.9/240.9 kB\u001b[0m \u001b[31m5.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Building wheels for collected packages: emoji\n",
            "  Building wheel for emoji (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for emoji: filename=emoji-2.2.0-py3-none-any.whl size=234926 sha256=7f2893be928828b89894ff207a194df428ac918037c861cb83e9c30326572521\n",
            "  Stored in directory: /root/.cache/pip/wheels/02/3d/88/51a592b9ad17e7899126563698b4e3961983ebe85747228ba6\n",
            "Successfully built emoji\n",
            "Installing collected packages: emoji\n",
            "Successfully installed emoji-2.2.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import emoji\n",
        "print(emoji.demojize(\"java is 😋\"))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qSo-mIqkOP3y",
        "outputId": "02cac461-158e-4b55-d4f6-12733b013c89"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "java is :face_savoring_food:\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Tokenization**"
      ],
      "metadata": {
        "id": "OqM0t8BuO3cX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# 1.Using split function\n",
        "# word tokenization\n",
        "sent1 = \"i am in hyd\"\n",
        "sent1.split()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iKa7rXgrOeON",
        "outputId": "a2ab0f5e-9207-430c-e848-770a20039f75"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['i', 'am', 'in', 'hyd']"
            ]
          },
          "metadata": {},
          "execution_count": 84
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# sentence tokenization\n",
        "\n",
        "sent2 = \"i im in hyd.Tommorrow i will go to my home\"\n",
        "sent2.split('.')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lbqO68rEQsgJ",
        "outputId": "ff491a23-5746-444c-cdef-74eef39fe0c6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['i im in hyd', 'Tommorrow i will go to my home']"
            ]
          },
          "metadata": {},
          "execution_count": 86
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# problems with split\n",
        "\n",
        "sent3 = \"i am going to hyd!!\"\n",
        "sent3.split()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w5xyI3Z8Q-y4",
        "outputId": "f41ca709-2a6b-4622-e23d-dd9d5a45f15c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['i', 'am', 'going', 'to', 'hyd!!']"
            ]
          },
          "metadata": {},
          "execution_count": 88
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sent4 = \"I am in U.S\"\n",
        "sent4.split('.')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kPh38m7pRKMz",
        "outputId": "c522d878-864e-49ee-97f7-fb0aba7712f9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['I am in U', 'S']"
            ]
          },
          "metadata": {},
          "execution_count": 89
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 2 regular expression\n",
        "import re\n",
        "sent3 = \"I am in hyd\"\n",
        "tokens = re.findall(\"[\\w']+\",sent3)\n",
        "tokens"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qw5qAOYKRXCA",
        "outputId": "80c75857-7a62-44dd-f650-fe541325b6cc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['I', 'am', 'in', 'hyd']"
            ]
          },
          "metadata": {},
          "execution_count": 91
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "text = \"\"\"Lorem ipsum dolor sit amet, consectetur adipiscing elit.\n",
        " Nunc et aliquam metus, sed pellentesque augue. Donec auctor nisl quis eros dictum,\n",
        "  ac aliquam nisi venenatis. Fusce semper orci a urna aliquet, eu dignissim massa egestas.\n",
        "  Sed tincidunt dolor sit amet nisl vestibulum, ac dapibus odio consequat.\n",
        "  Etiam rutrum metus eu sapien pellentesque, a faucibus leo ultrices.\n",
        "  Praesent et dui sed augue sagittis interdum a ac quam.\n",
        "  Sed eget nisi ac augue maximus pretium nec in metus.\"\"\"\n",
        "sentences = re.compile('[.!?]').split(text)\n",
        "sentences"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yC4UGt5xVhT8",
        "outputId": "eb10a4ac-6102-491d-cca3-e2b2598e5a30"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Lorem ipsum dolor sit amet, consectetur adipiscing elit',\n",
              " '\\n Nunc et aliquam metus, sed pellentesque augue',\n",
              " ' Donec auctor nisl quis eros dictum,\\n  ac aliquam nisi venenatis',\n",
              " ' Fusce semper orci a urna aliquet, eu dignissim massa egestas',\n",
              " ' \\n  Sed tincidunt dolor sit amet nisl vestibulum, ac dapibus odio consequat',\n",
              " ' \\n  Etiam rutrum metus eu sapien pellentesque, a faucibus leo ultrices',\n",
              " ' \\n  Praesent et dui sed augue sagittis interdum a ac quam',\n",
              " '\\n Sed eget nisi ac augue maximus pretium nec in metus',\n",
              " '']"
            ]
          },
          "metadata": {},
          "execution_count": 95
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# NLTK\n",
        "from nltk.tokenize import word_tokenize,sent_tokenize"
      ],
      "metadata": {
        "id": "QFfi1iTRWFKq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "nltk.download()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iQ988HlBZO8x",
        "outputId": "fe07dca7-f7cc-4e5f-c5a6-f141436ab466"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "NLTK Downloader\n",
            "---------------------------------------------------------------------------\n",
            "    d) Download   l) List    u) Update   c) Config   h) Help   q) Quit\n",
            "---------------------------------------------------------------------------\n",
            "Downloader> d\n",
            "\n",
            "Download which package (l=list; x=cancel)?\n",
            "  Identifier> l\n",
            "Packages:\n",
            "  [ ] abc................. Australian Broadcasting Commission 2006\n",
            "  [ ] alpino.............. Alpino Dutch Treebank\n",
            "  [ ] averaged_perceptron_tagger Averaged Perceptron Tagger\n",
            "  [ ] averaged_perceptron_tagger_ru Averaged Perceptron Tagger (Russian)\n",
            "  [ ] basque_grammars..... Grammars for Basque\n",
            "  [ ] bcp47............... BCP-47 Language Tags\n",
            "  [ ] biocreative_ppi..... BioCreAtIvE (Critical Assessment of Information\n",
            "                           Extraction Systems in Biology)\n",
            "  [ ] bllip_wsj_no_aux.... BLLIP Parser: WSJ Model\n",
            "  [ ] book_grammars....... Grammars from NLTK Book\n",
            "  [ ] brown............... Brown Corpus\n",
            "  [ ] brown_tei........... Brown Corpus (TEI XML Version)\n",
            "  [ ] cess_cat............ CESS-CAT Treebank\n",
            "  [ ] cess_esp............ CESS-ESP Treebank\n",
            "  [ ] chat80.............. Chat-80 Data Files\n",
            "  [ ] city_database....... City Database\n",
            "  [ ] cmudict............. The Carnegie Mellon Pronouncing Dictionary (0.6)\n",
            "  [ ] comparative_sentences Comparative Sentence Dataset\n",
            "  [ ] comtrans............ ComTrans Corpus Sample\n",
            "  [ ] conll2000........... CONLL 2000 Chunking Corpus\n",
            "Hit Enter to continue: \n",
            "Hit Enter to continue: \n",
            "  [ ] conll2002........... CONLL 2002 Named Entity Recognition Corpus\n",
            "  [ ] conll2007........... Dependency Treebanks from CoNLL 2007 (Catalan\n",
            "                           and Basque Subset)\n",
            "  [ ] crubadan............ Crubadan Corpus\n",
            "  [ ] dependency_treebank. Dependency Parsed Treebank\n",
            "  [ ] dolch............... Dolch Word List\n",
            "  [ ] europarl_raw........ Sample European Parliament Proceedings Parallel\n",
            "                           Corpus\n",
            "  [ ] extended_omw........ Extended Open Multilingual WordNet\n",
            "  [ ] floresta............ Portuguese Treebank\n",
            "  [ ] framenet_v15........ FrameNet 1.5\n",
            "  [ ] framenet_v17........ FrameNet 1.7\n",
            "  [ ] gazetteers.......... Gazeteer Lists\n",
            "  [ ] genesis............. Genesis Corpus\n",
            "  [ ] gutenberg........... Project Gutenberg Selections\n",
            "  [ ] ieer................ NIST IE-ER DATA SAMPLE\n",
            "  [ ] inaugural........... C-Span Inaugural Address Corpus\n",
            "  [ ] indian.............. Indian Language POS-Tagged Corpus\n",
            "  [ ] jeita............... JEITA Public Morphologically Tagged Corpus (in\n",
            "                           ChaSen format)\n",
            "  [ ] kimmo............... PC-KIMMO Data Files\n",
            "  [ ] knbc................ KNB Corpus (Annotated blog corpus)\n",
            "  [ ] large_grammars...... Large context-free and feature-based grammars\n",
            "                           for parser comparison\n",
            "  [ ] lin_thesaurus....... Lin's Dependency Thesaurus\n",
            "  [ ] mac_morpho.......... MAC-MORPHO: Brazilian Portuguese news text with\n",
            "                           part-of-speech tags\n",
            "  [ ] machado............. Machado de Assis -- Obra Completa\n",
            "  [ ] masc_tagged......... MASC Tagged Corpus\n",
            "  [ ] maxent_ne_chunker... ACE Named Entity Chunker (Maximum entropy)\n",
            "  [ ] maxent_treebank_pos_tagger Treebank Part of Speech Tagger (Maximum entropy)\n",
            "  [ ] moses_sample........ Moses Sample Models\n",
            "  [ ] movie_reviews....... Sentiment Polarity Dataset Version 2.0\n",
            "  [ ] mte_teip5........... MULTEXT-East 1984 annotated corpus 4.0\n",
            "  [ ] mwa_ppdb............ The monolingual word aligner (Sultan et al.\n",
            "                           2015) subset of the Paraphrase Database.\n",
            "  [ ] names............... Names Corpus, Version 1.3 (1994-03-29)\n",
            "  [ ] nombank.1.0......... NomBank Corpus 1.0\n",
            "  [ ] nonbreaking_prefixes Non-Breaking Prefixes (Moses Decoder)\n",
            "  [ ] nps_chat............ NPS Chat\n",
            "  [ ] omw-1.4............. Open Multilingual Wordnet\n",
            "  [ ] omw................. Open Multilingual Wordnet\n",
            "Hit Enter to continue: \n",
            "  [ ] opinion_lexicon..... Opinion Lexicon\n",
            "  [ ] panlex_swadesh...... PanLex Swadesh Corpora\n",
            "  [ ] paradigms........... Paradigm Corpus\n",
            "  [ ] pe08................ Cross-Framework and Cross-Domain Parser\n",
            "                           Evaluation Shared Task\n",
            "  [ ] perluniprops........ perluniprops: Index of Unicode Version 7.0.0\n",
            "                           character properties in Perl\n",
            "  [ ] pil................. The Patient Information Leaflet (PIL) Corpus\n",
            "  [ ] pl196x.............. Polish language of the XX century sixties\n",
            "  [ ] porter_test......... Porter Stemmer Test Files\n",
            "  [ ] ppattach............ Prepositional Phrase Attachment Corpus\n",
            "  [ ] problem_reports..... Problem Report Corpus\n",
            "  [ ] product_reviews_1... Product Reviews (5 Products)\n",
            "  [ ] product_reviews_2... Product Reviews (9 Products)\n",
            "  [ ] propbank............ Proposition Bank Corpus 1.0\n",
            "  [ ] pros_cons........... Pros and Cons\n",
            "  [ ] ptb................. Penn Treebank\n",
            "  [ ] punkt............... Punkt Tokenizer Models\n",
            "  [ ] qc.................. Experimental Data for Question Classification\n",
            "  [ ] reuters............. The Reuters-21578 benchmark corpus, ApteMod\n",
            "                           version\n",
            "Hit Enter to continue: \n",
            "  [ ] rslp................ RSLP Stemmer (Removedor de Sufixos da Lingua\n",
            "                           Portuguesa)\n",
            "  [ ] rte................. PASCAL RTE Challenges 1, 2, and 3\n",
            "  [ ] sample_grammars..... Sample Grammars\n",
            "  [ ] semcor.............. SemCor 3.0\n",
            "  [ ] senseval............ SENSEVAL 2 Corpus: Sense Tagged Text\n",
            "  [ ] sentence_polarity... Sentence Polarity Dataset v1.0\n",
            "  [ ] sentiwordnet........ SentiWordNet\n",
            "  [ ] shakespeare......... Shakespeare XML Corpus Sample\n",
            "  [ ] sinica_treebank..... Sinica Treebank Corpus Sample\n",
            "  [ ] smultron............ SMULTRON Corpus Sample\n",
            "  [ ] snowball_data....... Snowball Data\n",
            "  [ ] spanish_grammars.... Grammars for Spanish\n",
            "  [ ] state_union......... C-Span State of the Union Address Corpus\n",
            "  [ ] subjectivity........ Subjectivity Dataset v1.0\n",
            "  [ ] swadesh............. Swadesh Wordlists\n",
            "  [ ] switchboard......... Switchboard Corpus Sample\n",
            "  [ ] tagsets............. Help on Tagsets\n",
            "  [ ] timit............... TIMIT Corpus Sample\n",
            "  [ ] toolbox............. Toolbox Sample Files\n",
            "  [ ] treebank............ Penn Treebank Sample\n",
            "Hit Enter to continue: \n",
            "  [ ] twitter_samples..... Twitter Samples\n",
            "  [ ] udhr2............... Universal Declaration of Human Rights Corpus\n",
            "                           (Unicode Version)\n",
            "  [ ] udhr................ Universal Declaration of Human Rights Corpus\n",
            "  [ ] unicode_samples..... Unicode Samples\n",
            "  [ ] universal_tagset.... Mappings to the Universal Part-of-Speech Tagset\n",
            "  [ ] universal_treebanks_v20 Universal Treebanks Version 2.0\n",
            "  [ ] vader_lexicon....... VADER Sentiment Lexicon\n",
            "  [ ] verbnet3............ VerbNet Lexicon, Version 3.3\n",
            "  [ ] verbnet............. VerbNet Lexicon, Version 2.1\n",
            "  [ ] webtext............. Web Text Corpus\n",
            "  [ ] wmt15_eval.......... Evaluation data from WMT15\n",
            "  [ ] word2vec_sample..... Word2Vec Sample\n",
            "  [ ] wordnet2021......... Open English Wordnet 2021\n",
            "  [ ] wordnet2022......... Open English Wordnet 2022\n",
            "  [ ] wordnet31........... Wordnet 3.1\n",
            "  [ ] wordnet............. WordNet\n",
            "  [ ] wordnet_ic.......... WordNet-InfoContent\n",
            "  [ ] words............... Word Lists\n",
            "  [ ] ycoe................ York-Toronto-Helsinki Parsed Corpus of Old\n",
            "                           English Prose\n",
            "Hit Enter to continue: \n",
            "\n",
            "Collections:\n",
            "  [P] all-corpora......... All the corpora\n",
            "  [P] all-nltk............ All packages available on nltk_data gh-pages\n",
            "                           branch\n",
            "  [P] all................. All packages\n",
            "  [P] book................ Everything used in the NLTK Book\n",
            "  [P] popular............. Popular packages\n",
            "  [ ] tests............... Packages for running tests\n",
            "  [ ] third-party......... Third-party data packages\n",
            "\n",
            "([*] marks installed packages; [P] marks partially installed collections)\n",
            "\n",
            "Download which package (l=list; x=cancel)?\n",
            "  Identifier> x\n",
            "\n",
            "---------------------------------------------------------------------------\n",
            "    d) Download   l) List    u) Update   c) Config   h) Help   q) Quit\n",
            "---------------------------------------------------------------------------\n",
            "Downloader> d\n",
            "\n",
            "Download which package (l=list; x=cancel)?\n",
            "  Identifier> x\n",
            "\n",
            "---------------------------------------------------------------------------\n",
            "    d) Download   l) List    u) Update   c) Config   h) Help   q) Quit\n",
            "---------------------------------------------------------------------------\n",
            "Downloader> q\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 100
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "nltk.download('punkt')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OW6Nf7fPZcHS",
        "outputId": "47a55cb3-0262-47e0-f255-d5d3e666b3b5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 102
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sent1 = \"i am going to visit delhi!\"\n",
        "word_tokenize(sent1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uVcC0gpnXYBa",
        "outputId": "3724199d-8770-4788-df2e-52e871132f8e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['i', 'am', 'going', 'to', 'visit', 'delhi', '!']"
            ]
          },
          "metadata": {},
          "execution_count": 103
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sent5 = \"I have a Ph.D in A.I\""
      ],
      "metadata": {
        "id": "-nRlYOZlYIBQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "word_tokenize(sent5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iu7EFPaHZs2I",
        "outputId": "632e689e-c67a-4baf-f965-feea9c5b223f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['I', 'have', 'a', 'Ph.D', 'in', 'A.I']"
            ]
          },
          "metadata": {},
          "execution_count": 106
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sent6 = \"we're here to help! mail us at pavan@gmail.com\""
      ],
      "metadata": {
        "id": "ANuVb9ZPZwr9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "word_tokenize(sent6)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iK3NiEVpaC3p",
        "outputId": "71be097c-f09b-4131-e3ff-c244d58519bd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['we',\n",
              " \"'re\",\n",
              " 'here',\n",
              " 'to',\n",
              " 'help',\n",
              " '!',\n",
              " 'mail',\n",
              " 'us',\n",
              " 'at',\n",
              " 'pavan',\n",
              " '@',\n",
              " 'gmail.com']"
            ]
          },
          "metadata": {},
          "execution_count": 113
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sent7 = \"A 5km ride cost $10.50\""
      ],
      "metadata": {
        "id": "3OoQwtdkaE9r"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "word_tokenize(sent7)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I6VE7xPWaQ_I",
        "outputId": "ba442283-0a22-4cca-c58b-d8843c639388"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['A', '5km', 'ride', 'cost', '$', '10.50']"
            ]
          },
          "metadata": {},
          "execution_count": 116
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Spacy\n",
        "import spacy"
      ],
      "metadata": {
        "id": "J6RE-g2haTWz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "nlp = spacy.load(\"en_core_web_sm\")"
      ],
      "metadata": {
        "id": "RLDDNXt9advt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "doc1 = nlp(sent5)\n",
        "doc2 = nlp(sent6)\n",
        "doc3 = nlp(sent7)\n",
        "doc4 = nlp(sent1)"
      ],
      "metadata": {
        "id": "mQmG1WHJaoMS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for token in doc1:\n",
        "  print(token)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PI6knZ_Gay15",
        "outputId": "ee75c71c-9570-4698-af9d-eb8de7fa5803"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "I\n",
            "have\n",
            "a\n",
            "Ph\n",
            ".\n",
            "D\n",
            "in\n",
            "A.I\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for token in doc2:\n",
        "  print(token)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mwzYUKS5a5pd",
        "outputId": "84802b9f-d93f-4266-fb80-b5b80ca8c950"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "we\n",
            "'re\n",
            "here\n",
            "to\n",
            "help\n",
            "!\n",
            "mail\n",
            "us\n",
            "at\n",
            "pavan@gmail.com\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for token in doc3:\n",
        "  print(token)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ccXVInvFbAQY",
        "outputId": "f52c8e3e-dda8-4aac-9a45-88aaacef58e6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "A\n",
            "5\n",
            "km\n",
            "ride\n",
            "cost\n",
            "$\n",
            "10.50\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for token in doc4:\n",
        "  print(token)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JwkcE_nkbDn9",
        "outputId": "9a49d960-a4a2-4a14-97b1-a6736cc6ce83"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "i\n",
            "am\n",
            "going\n",
            "to\n",
            "visit\n",
            "delhi\n",
            "!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Stemming**"
      ],
      "metadata": {
        "id": "ynU4mMvNbZxi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# bring back to root form eg. play,played,playing -------->play\n",
        "\n",
        "from nltk.stem.porter import PorterStemmer"
      ],
      "metadata": {
        "id": "9ScfI9VkbNyV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ps = PorterStemmer()\n",
        "def stem_words(text):\n",
        "  return \" \".join([ps.stem(word) for word in text.split()])"
      ],
      "metadata": {
        "id": "_OV8kUrgcoKs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sample = \"walk walks walking walked\"\n",
        "stem_words(sample)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "z9ZWj9COc1OH",
        "outputId": "c3e53b4f-6e88-4a44-f26b-77cb64a1e180"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'walk walk walk walk'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 130
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Lemmatization**"
      ],
      "metadata": {
        "id": "iHyNex1cdS6y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "nltk.download('wordnet')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E494VuQ7fQJ7",
        "outputId": "8085a974-9342-455b-f55f-1beb95ba4fa8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "\n",
        "wordnet_lemmatizer = WordNetLemmatizer()\n",
        "\n",
        "sentence = \"He was running and eating at same time.He has bad habit of swimming after playing long hours in the sun\"\n",
        "punctuations = \"?:!.,;\"\n",
        "sentence_words = nltk.word_tokenize(sentence)\n",
        "\n",
        "for word in sentence_words:\n",
        "  if word in punctuations:\n",
        "    sentence_words.remove(word)\n",
        "\n",
        "print(\"{0:20}{1:20}\".format(\"Word\",\"Lemma\"))\n",
        "for word in sentence_words:\n",
        "  print(\"{0:20}{1:20}\".format(word, wordnet_lemmatizer.lemmatize(word,pos='v'))) #pos -->parts of speech , here \"v\"-->verb  :here specifying verb to lemmatize\n",
        "\n"
      ],
      "metadata": {
        "id": "dXD3Op55c8aO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "B2pEFpyXewqP"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}