Part 8.4: NLPretext_library.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Part 8.4: NLPretext_library.ipynb",
      "provenance": [],
      "mount_file_id": "1E-WsrJ3oyBlAn-MIBwZodm_W-Cu7m79q",
      "authorship_tag": "ABX9TyPauuUZ4g1gUwEVQMHw0DFc",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gulabpatel/NLP_Basics/blob/main/Part%208.4%3A%20NLPretext_library.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2hARM1sWd5ii"
      },
      "source": [
        "!pip install nlpretext"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PwE0HH8seIUl"
      },
      "source": [
        "This library uses Spacy as tokenizer. Current models supported are en_core_web_sm and fr_core_news_sm. If not installed, run the following commands:\n",
        "\n",
        "pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.3.1/en_core_web_sm-2.3.1.tar.gz\n",
        "\n",
        "pip install https://github.com/explosion/spacy-models/releases/download/fr_core_news_sm-2.3.0/fr_core_news_sm-2.3.0.tar.gz\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Iu2v6AfIhwHY"
      },
      "source": [
        "!pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.3.1/en_core_web_sm-2.3.1.tar.gz\n",
        "\n",
        "!pip install https://github.com/explosion/spacy-models/releases/download/fr_core_news_sm-2.3.0/fr_core_news_sm-2.3.0.tar.gz"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5O9LS16feA6i",
        "outputId": "60cd392b-5ee4-42b2-8c66-e2af890bc19c"
      },
      "source": [
        "!pip install dask[complete]==2021.3.0"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: dask[complete]==2021.3.0 in /usr/local/lib/python3.7/dist-packages (2021.3.0)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from dask[complete]==2021.3.0) (3.13)\n",
            "Requirement already satisfied: cloudpickle>=0.2.2; extra == \"complete\" in /usr/local/lib/python3.7/dist-packages (from dask[complete]==2021.3.0) (1.3.0)\n",
            "Requirement already satisfied: partd>=0.3.10; extra == \"complete\" in /usr/local/lib/python3.7/dist-packages (from dask[complete]==2021.3.0) (1.2.0)\n",
            "Requirement already satisfied: distributed>=2021.03.0; extra == \"complete\" in /usr/local/lib/python3.7/dist-packages (from dask[complete]==2021.3.0) (2021.7.0)\n",
            "Requirement already satisfied: fsspec>=0.6.0; extra == \"complete\" in /usr/local/lib/python3.7/dist-packages (from dask[complete]==2021.3.0) (2021.7.0)\n",
            "Requirement already satisfied: numpy>=1.15.1; extra == \"complete\" in /usr/local/lib/python3.7/dist-packages (from dask[complete]==2021.3.0) (1.19.5)\n",
            "Requirement already satisfied: toolz>=0.8.2; extra == \"complete\" in /usr/local/lib/python3.7/dist-packages (from dask[complete]==2021.3.0) (0.11.1)\n",
            "Requirement already satisfied: pandas>=0.25.0; extra == \"complete\" in /usr/local/lib/python3.7/dist-packages (from dask[complete]==2021.3.0) (1.1.5)\n",
            "Requirement already satisfied: bokeh!=2.0.0,>=1.0.0; extra == \"complete\" in /usr/local/lib/python3.7/dist-packages (from dask[complete]==2021.3.0) (2.3.3)\n",
            "Requirement already satisfied: locket in /usr/local/lib/python3.7/dist-packages (from partd>=0.3.10; extra == \"complete\"->dask[complete]==2021.3.0) (0.2.1)\n",
            "Requirement already satisfied: msgpack>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from distributed>=2021.03.0; extra == \"complete\"->dask[complete]==2021.3.0) (1.0.2)\n",
            "Requirement already satisfied: click>=6.6 in /usr/local/lib/python3.7/dist-packages (from distributed>=2021.03.0; extra == \"complete\"->dask[complete]==2021.3.0) (7.1.2)\n",
            "Requirement already satisfied: tornado>=5; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from distributed>=2021.03.0; extra == \"complete\"->dask[complete]==2021.3.0) (5.1.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from distributed>=2021.03.0; extra == \"complete\"->dask[complete]==2021.3.0) (57.0.0)\n",
            "Requirement already satisfied: zict>=0.1.3 in /usr/local/lib/python3.7/dist-packages (from distributed>=2021.03.0; extra == \"complete\"->dask[complete]==2021.3.0) (2.0.0)\n",
            "Requirement already satisfied: sortedcontainers!=2.0.0,!=2.0.1 in /usr/local/lib/python3.7/dist-packages (from distributed>=2021.03.0; extra == \"complete\"->dask[complete]==2021.3.0) (2.4.0)\n",
            "Requirement already satisfied: tblib>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from distributed>=2021.03.0; extra == \"complete\"->dask[complete]==2021.3.0) (1.7.0)\n",
            "Requirement already satisfied: psutil>=5.0 in /usr/local/lib/python3.7/dist-packages (from distributed>=2021.03.0; extra == \"complete\"->dask[complete]==2021.3.0) (5.4.8)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.25.0; extra == \"complete\"->dask[complete]==2021.3.0) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.25.0; extra == \"complete\"->dask[complete]==2021.3.0) (2018.9)\n",
            "Requirement already satisfied: packaging>=16.8 in /usr/local/lib/python3.7/dist-packages (from bokeh!=2.0.0,>=1.0.0; extra == \"complete\"->dask[complete]==2021.3.0) (21.0)\n",
            "Requirement already satisfied: Jinja2>=2.9 in /usr/local/lib/python3.7/dist-packages (from bokeh!=2.0.0,>=1.0.0; extra == \"complete\"->dask[complete]==2021.3.0) (2.11.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from bokeh!=2.0.0,>=1.0.0; extra == \"complete\"->dask[complete]==2021.3.0) (3.7.4.3)\n",
            "Requirement already satisfied: pillow>=7.1.0 in /usr/local/lib/python3.7/dist-packages (from bokeh!=2.0.0,>=1.0.0; extra == \"complete\"->dask[complete]==2021.3.0) (7.1.2)\n",
            "Requirement already satisfied: heapdict in /usr/local/lib/python3.7/dist-packages (from zict>=0.1.3->distributed>=2021.03.0; extra == \"complete\"->dask[complete]==2021.3.0) (1.0.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas>=0.25.0; extra == \"complete\"->dask[complete]==2021.3.0) (1.15.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=16.8->bokeh!=2.0.0,>=1.0.0; extra == \"complete\"->dask[complete]==2021.3.0) (2.4.7)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from Jinja2>=2.9->bokeh!=2.0.0,>=1.0.0; extra == \"complete\"->dask[complete]==2021.3.0) (2.0.1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bMEi441seWR2",
        "outputId": "e78f3c9b-72f5-4869-f9ba-85bcda1044d3"
      },
      "source": [
        "from nlpretext import Preprocessor\n",
        "text = \"I just got the best dinner in my life @latourdargent !!! I  recommend 😀 #food #paris \\n\"\n",
        "preprocessor = Preprocessor()\n",
        "text = preprocessor.run(text)\n",
        "print(text)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "I just got the best dinner in my life !!! I recommend\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g-K8uNRnepvc"
      },
      "source": [
        "Another possibility is to create your custom pipeline if you know exactly what function to apply on your data, here's an example:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PCHLXZJkeqgt",
        "outputId": "a2a48cf8-8d91-4641-ea28-6fdb7bf7537b"
      },
      "source": [
        "from nlpretext import Preprocessor\n",
        "from nlpretext.basic.preprocess import (normalize_whitespace, remove_punct, remove_eol_characters,\n",
        "remove_stopwords, lower_text)\n",
        "from nlpretext.social.preprocess import remove_mentions, remove_hashtag, remove_emoji\n",
        "text = \"I just got the best dinner in my life @latourdargent !!! I  recommend 😀 #food #paris \\n\"\n",
        "preprocessor = Preprocessor()\n",
        "preprocessor.pipe(lower_text)\n",
        "preprocessor.pipe(remove_mentions)\n",
        "preprocessor.pipe(remove_hashtag)\n",
        "preprocessor.pipe(remove_emoji)\n",
        "preprocessor.pipe(remove_eol_characters)\n",
        "preprocessor.pipe(remove_stopwords, args={'lang': 'en'})\n",
        "preprocessor.pipe(remove_punct)\n",
        "preprocessor.pipe(normalize_whitespace)\n",
        "text = preprocessor.run(text)\n",
        "print(text)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "dinner life recommend\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ljpf4Fgwfhao"
      },
      "source": [
        "**Load text data**\n",
        "\n",
        "Pre-processing text data is useful only if you have loaded data to process! Importing text data as strings in your code can be really simple if you have short texts contained in a local .txt, but it can quickly become difficult if you want to load a lot of texts, stored in multiple formats and divided in multiple files. Hopefully, you can use NLPretext's TextLoader class to easily import text data. Our TextLoader class makes use of Dask, so be sure to install the library if you want to use it, as mentioned above."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AJ1zU3fTfHBy"
      },
      "source": [
        "from nlpretext.textloader import TextLoader\n",
        "files_path = \"/content/drive/MyDrive/Colab Notebooks/NLP/Text Files/char_sequences.txt\"\n",
        "text_loader = TextLoader()\n",
        "text_dataframe = text_loader.read_text(files_path)\n",
        "print(text_dataframe.text.values.tolist())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y3pU44rgiY_B"
      },
      "source": [
        "**Replacing emails**\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5uIcBk9WgXn1",
        "outputId": "cb8fd109-9288-441b-d969-149f50e57dcc"
      },
      "source": [
        "from nlpretext.basic.preprocess import replace_emails\n",
        "example = \"I have forwarded this email to obama@whitehouse.gov\"\n",
        "example = replace_emails(example, replace_with=\"*EMAIL*\")\n",
        "print(example)\n",
        "\n",
        "# \"I have forwarded this email to *EMAIL*\""
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "I have forwarded this email to *EMAIL*\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sxv1BI_9im_L"
      },
      "source": [
        "**Replacing phone numbers**\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N6NZx3XCihIi",
        "outputId": "cae3ab5a-033f-4b54-8814-87243c55e8b8"
      },
      "source": [
        "from nlpretext.basic.preprocess import replace_phone_numbers\n",
        "example = \"My phone number is 0606060606\"\n",
        "example = replace_phone_numbers(example, country_to_detect=[\"FR\"], replace_with=\"*PHONE*\")\n",
        "print(example)\n",
        "# \"My phone number is *PHONE*\""
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "My phone number is *PHONE*\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5qeSS7Lwit97"
      },
      "source": [
        "**Removing Hashtags**\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pj7sX-itiqlp",
        "outputId": "c9687094-1758-4ce2-c451-c285da069998"
      },
      "source": [
        "from nlpretext.social.preprocess import remove_hashtag\n",
        "example = \"This restaurant was amazing #food #foodie #foodstagram #dinner\"\n",
        "example = remove_hashtag(example)\n",
        "print(example)\n",
        "# \"This restaurant was amazing\""
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "This restaurant was amazing\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "27COWEnSi2wa"
      },
      "source": [
        "**Extracting emojis**\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q73rzBjViyFr",
        "outputId": "e8ec8f19-ce63-4749-e31f-9c4330d306c6"
      },
      "source": [
        "from nlpretext.social.preprocess import extract_emojis\n",
        "example = \"I take care of my skin 😀\"\n",
        "example = extract_emojis(example)\n",
        "print(example)\n",
        "# [':grinning_face:']"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[':grinning_face:']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_iJwgPPojCjd"
      },
      "source": [
        "**Data augmentation**\n",
        "\n",
        "The augmentation module helps you to generate new texts based on your given examples by modifying some words in the initial ones and to keep associated entities unchanged, if any, in the case of NER tasks. If you want words other than entities to remain unchanged, you can specify it within the stopwords argument. Modifications depend on the chosen method, the ones currently supported by the module are substitutions with synonyms using Wordnet or BERT from the nlpaug library."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "o_R2v0I1i6iY",
        "outputId": "13c3eca4-3925-4f14-9d41-030896197b74"
      },
      "source": [
        "from nlpretext.augmentation.text_augmentation import augment_text\n",
        "example = \"I want to buy a small black handbag please.\"\n",
        "entities = [{'entity': 'Color', 'word': 'black', 'startCharIndex': 22, 'endCharIndex': 27}]\n",
        "example = augment_text(example, method=\"wordnet_synonym\", entities=entities)\n",
        "print(example)\n",
        "# \"I need to buy a small black pocketbook please.\""
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/wordnet.zip.\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "('Unity want to buy a modest black handbag please.', [{'entity': 'Color', 'startCharIndex': 27, 'endCharIndex': 32, 'word': 'black'}])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z-pPgeHJjJrF"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}