article_spinner.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Untitled4.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyNnZf/PCqznDM1uOkn6wYGS",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/dvircohen0/NLP/blob/main/article_spinner.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0BxlRmLO6hWM",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "417e0fcc-cf3f-4b48-b0ce-b809087f72a4"
      },
      "source": [
        "import nltk\n",
        "import numpy as np\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn.neural_network import MLPClassifier\n",
        "from sklearn import svm\n",
        "from bs4 import BeautifulSoup\n",
        "from nltk.corpus import stopwords\n",
        "from sklearn.model_selection import train_test_split\n",
        "from IPython.display import HTML as html_print\n",
        "from IPython.display import display\n",
        "import re \n",
        "import random\n",
        "import textwrap\n",
        "nltk.download('punkt')"
      ],
      "execution_count": 119,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 119
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cI3y3KQrRuCd",
        "outputId": "a256cda2-6e59-46ed-dd00-5c07896ecd46"
      },
      "source": [
        "!wget https://raw.githubusercontent.com/AmirYunus/py_sentiment_analysis/9f0a92edc228d6af49f9bbeb66dcc80e8b412e29/electronics/positive.review\n",
        "\n",
        "positive_reviews=[]\n",
        "pos_rev =  BeautifulSoup(open('/content/positive.review').read())\n",
        "for p in pos_rev.findAll('review_text'):\n",
        "  positive_reviews.append(p.text.strip())"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-05-19 10:59:06--  https://raw.githubusercontent.com/AmirYunus/py_sentiment_analysis/9f0a92edc228d6af49f9bbeb66dcc80e8b412e29/electronics/positive.review\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1105010 (1.1M) [text/plain]\n",
            "Saving to: ‘positive.review’\n",
            "\n",
            "positive.review     100%[===================>]   1.05M  --.-KB/s    in 0.06s   \n",
            "\n",
            "2021-05-19 10:59:06 (16.6 MB/s) - ‘positive.review’ saved [1105010/1105010]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I3L3Tvr1SESQ"
      },
      "source": [
        "def random_sample(d):\n",
        "    r = random.random()\n",
        "    cumulative = 0\n",
        "    for w, p in d.items():\n",
        "        cumulative += p\n",
        "        if r <   cumulative:\n",
        "            return w"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xOHBpTpd6nN2"
      },
      "source": [
        "Word_lemmatizer = WordNetLemmatizer()\n",
        "\n",
        "trigram ={}\n",
        "for review in positive_reviews:\n",
        "    s=review.lower()\n",
        "    tokens = nltk.tokenize.word_tokenize(s)\n",
        "    for i in range(1, len(tokens)-1):\n",
        "        if (tokens[i-1] , tokens[i+1]) in trigram:\n",
        "            if  tokens[i] in trigram[(tokens[i-1] , tokens[i+1])]:\n",
        "                trigram[(tokens[i-1] , tokens[i+1])][tokens[i]] += 1\n",
        "            else: trigram[(tokens[i-1] , tokens[i+1])][tokens[i]] = 1\n",
        "        else: trigram[(tokens[i-1] , tokens[i+1])] = {tokens[i] : 1}\n",
        "            \n",
        "\n",
        "for key  in list(trigram.keys()):\n",
        "    if len(trigram[key]) > 1:\n",
        "        factor = 1/sum(trigram[key].values())\n",
        "        for t in list(trigram[key].keys()):\n",
        "            trigram[key][t]*=factor    \n",
        "    else: del(trigram[key])"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fcP8JnzmUr79"
      },
      "source": [
        "def cstr(s, color='black'):\n",
        "    return \"<text style=color:{}>{}</text>\".format(color, s)\n",
        "\n",
        "def print_color(t):\n",
        "    display(html_print(' '.join([cstr(ti, color=ci) for ti,ci in t])))"
      ],
      "execution_count": 120,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v0klROAwejSj"
      },
      "source": [
        "def test_spinner():\n",
        "    review = random.choice(positive_reviews)\n",
        "    s = review.lower()\n",
        "    tokens = nltk.tokenize.word_tokenize(s)\n",
        "\n",
        "    colored_true=[0]*len(tokens)\n",
        "    colored_true[0] = (tokens[0],'black')\n",
        "    colored_true[-1] = (tokens[-1],'black')\n",
        "\n",
        "    colored_pred=[0]*len(tokens)\n",
        "    colored_pred[0] = (tokens[0],'black')\n",
        "    colored_pred[-1] = (tokens[-1],'black')\n",
        "\n",
        "    for i in range(1, len(tokens)-1):\n",
        "        colored_pred[i]=(tokens[i],'black')\n",
        "        colored_true[i]=(tokens[i],'black')\n",
        "        if random.random() < 0.2: # 20% chance of replacement\n",
        "            k = (tokens[i-1], tokens[i+1])\n",
        "            if k in trigram:\n",
        "\n",
        "                colored_true[i] = (tokens[i],'red')\n",
        "                w = random_sample(trigram[k])\n",
        "                tokens[i] = w\n",
        "                colored_pred[i] = (w,'red')\n",
        "                \n",
        "    print(\"Original text: \\n\")\n",
        "    print_color(colored_true)\n",
        "    print(\"\\n Spun text: \\n\")\n",
        "    print_color(colored_pred)\n",
        "    \n"
      ],
      "execution_count": 121,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 272
        },
        "id": "4pCuXGxQ6QBt",
        "outputId": "2e5920a0-9273-4d9b-af03-eb45e9b84c9c"
      },
      "source": [
        "test_spinner()"
      ],
      "execution_count": 122,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Original text: \n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<text style=color:black>i</text> <text style=color:black>had</text> <text style=color:black>a</text> <text style=color:black>small</text> <text style=color:black>512</text> <text style=color:black>sandisk</text> <text style=color:black>a</text> <text style=color:black>while</text> <text style=color:black>back</text> <text style=color:black>,</text> <text style=color:black>so</text> <text style=color:black>when</text> <text style=color:black>i</text> <text style=color:black>saw</text> <text style=color:black>a</text> <text style=color:black>review</text> <text style=color:red>for</text> <text style=color:black>the</text> <text style=color:black>titanium</text> <text style=color:black>2gb</text> <text style=color:black>cruzer</text> <text style=color:black>in</text> <text style=color:black>a</text> <text style=color:black>popular</text> <text style=color:black>computer</text> <text style=color:black>mag</text> <text style=color:black>,</text> <text style=color:black>i</text> <text style=color:black>decided</text> <text style=color:black>to</text> <text style=color:black>upgrade</text> <text style=color:black>from</text> <text style=color:black>my</text> <text style=color:black>lexar</text> <text style=color:black>1gb</text> <text style=color:black>.</text> <text style=color:black>so</text> <text style=color:red>far</text> <text style=color:black>,</text> <text style=color:black>i</text> <text style=color:black>'m</text> <text style=color:black>very</text> <text style=color:black>impressed</text> <text style=color:black>.</text> <text style=color:black>just</text> <text style=color:black>did</text> <text style=color:black>a</text> <text style=color:black>1.26gb</text> <text style=color:black>transfer</text> <text style=color:black>that</text> <text style=color:black>took</text> <text style=color:black>140</text> <text style=color:black>seconds</text> <text style=color:black>.</text> <text style=color:red>i</text> <text style=color:red>was</text> <text style=color:black>a</text> <text style=color:black>big</text> <text style=color:black>leery</text> <text style=color:black>of</text> <text style=color:black>the</text> <text style=color:red>included</text> <text style=color:black>software</text> <text style=color:black>,</text> <text style=color:black>but</text> <text style=color:black>was</text> <text style=color:black>pleased</text> <text style=color:black>to</text> <text style=color:black>see</text> <text style=color:black>that</text> <text style=color:black>if</text> <text style=color:red>i</text> <text style=color:black>did</text> <text style=color:black>n't</text> <text style=color:black>care</text> <text style=color:black>for</text> <text style=color:black>it</text> <text style=color:black>,</text> <text style=color:black>it</text> <text style=color:black>was</text> <text style=color:black>easily</text> <text style=color:red>removed</text> <text style=color:black>.</text> <text style=color:red>i</text> <text style=color:red>have</text> <text style=color:black>not</text> <text style=color:black>had</text> <text style=color:black>an</text> <text style=color:black>issue</text> <text style=color:black>with</text> <text style=color:black>either</text> <text style=color:black>the</text> <text style=color:red>software</text> <text style=color:black>or</text> <text style=color:black>the</text> <text style=color:black>drive</text> <text style=color:red>,</text> <text style=color:black>and</text> <text style=color:black>am</text> <text style=color:red>very</text> <text style=color:black>impressed</text> <text style=color:red>with</text> <text style=color:black>the</text> <text style=color:black>design</text> <text style=color:black>of</text> <text style=color:black>the</text> <text style=color:red>retractable</text> <text style=color:black>interface</text> <text style=color:black>that</text> <text style=color:black>requires</text> <text style=color:black>pressure</text> <text style=color:black>to</text> <text style=color:red>expose</text> <text style=color:black>or</text> <text style=color:black>retract</text> <text style=color:black>.</text> <text style=color:black>this</text> <text style=color:black>is</text> <text style=color:black>a</text> <text style=color:black>very</text> <text style=color:black>good</text> <text style=color:black>drive</text> <text style=color:black>,</text> <text style=color:black>and</text> <text style=color:red>contrary</text> <text style=color:red>to</text> <text style=color:black>the</text> <text style=color:black>negative</text> <text style=color:black>comments</text> <text style=color:black>,</text> <text style=color:red>i</text> <text style=color:black>would</text> <text style=color:black>encourage</text> <text style=color:black>anyone</text> <text style=color:black>looking</text> <text style=color:black>for</text> <text style=color:black>a</text> <text style=color:black>fast</text> <text style=color:black>,</text> <text style=color:red>stylish</text> <text style=color:black>,</text> <text style=color:black>practical</text> <text style=color:black>jump</text> <text style=color:black>drive</text> <text style=color:black>to</text> <text style=color:black>seriously</text> <text style=color:black>conside</text> <text style=color:black>the</text> <text style=color:black>titanium</text> <text style=color:black>cruzer</text>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            " Spun text: \n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<text style=color:black>i</text> <text style=color:black>had</text> <text style=color:black>a</text> <text style=color:black>small</text> <text style=color:black>512</text> <text style=color:black>sandisk</text> <text style=color:black>a</text> <text style=color:black>while</text> <text style=color:black>back</text> <text style=color:black>,</text> <text style=color:black>so</text> <text style=color:black>when</text> <text style=color:black>i</text> <text style=color:black>saw</text> <text style=color:black>a</text> <text style=color:black>review</text> <text style=color:red>of</text> <text style=color:black>the</text> <text style=color:black>titanium</text> <text style=color:black>2gb</text> <text style=color:black>cruzer</text> <text style=color:black>in</text> <text style=color:black>a</text> <text style=color:black>popular</text> <text style=color:black>computer</text> <text style=color:black>mag</text> <text style=color:black>,</text> <text style=color:black>i</text> <text style=color:black>decided</text> <text style=color:black>to</text> <text style=color:black>upgrade</text> <text style=color:black>from</text> <text style=color:black>my</text> <text style=color:black>lexar</text> <text style=color:black>1gb</text> <text style=color:black>.</text> <text style=color:black>so</text> <text style=color:red>clean</text> <text style=color:black>,</text> <text style=color:black>i</text> <text style=color:black>'m</text> <text style=color:black>very</text> <text style=color:black>impressed</text> <text style=color:black>.</text> <text style=color:black>just</text> <text style=color:black>did</text> <text style=color:black>a</text> <text style=color:black>1.26gb</text> <text style=color:black>transfer</text> <text style=color:black>that</text> <text style=color:black>took</text> <text style=color:black>140</text> <text style=color:black>seconds</text> <text style=color:black>.</text> <text style=color:red>installation</text> <text style=color:red>was</text> <text style=color:black>a</text> <text style=color:black>big</text> <text style=color:black>leery</text> <text style=color:black>of</text> <text style=color:black>the</text> <text style=color:red>setpoint</text> <text style=color:black>software</text> <text style=color:black>,</text> <text style=color:black>but</text> <text style=color:black>was</text> <text style=color:black>pleased</text> <text style=color:black>to</text> <text style=color:black>see</text> <text style=color:black>that</text> <text style=color:black>if</text> <text style=color:red>i</text> <text style=color:black>did</text> <text style=color:black>n't</text> <text style=color:black>care</text> <text style=color:black>for</text> <text style=color:black>it</text> <text style=color:black>,</text> <text style=color:black>it</text> <text style=color:black>was</text> <text style=color:black>easily</text> <text style=color:red>reprogramable</text> <text style=color:black>.</text> <text style=color:red>i</text> <text style=color:red>had</text> <text style=color:black>not</text> <text style=color:black>had</text> <text style=color:black>an</text> <text style=color:black>issue</text> <text style=color:black>with</text> <text style=color:black>either</text> <text style=color:black>the</text> <text style=color:red>couch</text> <text style=color:black>or</text> <text style=color:black>the</text> <text style=color:black>drive</text> <text style=color:red>,</text> <text style=color:black>and</text> <text style=color:black>am</text> <text style=color:red>very</text> <text style=color:black>impressed</text> <text style=color:red>with</text> <text style=color:black>the</text> <text style=color:black>design</text> <text style=color:black>of</text> <text style=color:black>the</text> <text style=color:red>user</text> <text style=color:black>interface</text> <text style=color:black>that</text> <text style=color:black>requires</text> <text style=color:black>pressure</text> <text style=color:black>to</text> <text style=color:red>raise</text> <text style=color:black>or</text> <text style=color:black>retract</text> <text style=color:black>.</text> <text style=color:black>this</text> <text style=color:black>is</text> <text style=color:black>a</text> <text style=color:black>very</text> <text style=color:black>good</text> <text style=color:black>drive</text> <text style=color:black>,</text> <text style=color:black>and</text> <text style=color:red>listen</text> <text style=color:red>through</text> <text style=color:black>the</text> <text style=color:black>negative</text> <text style=color:black>comments</text> <text style=color:black>,</text> <text style=color:red>i</text> <text style=color:black>would</text> <text style=color:black>encourage</text> <text style=color:black>anyone</text> <text style=color:black>looking</text> <text style=color:black>for</text> <text style=color:black>a</text> <text style=color:black>fast</text> <text style=color:black>,</text> <text style=color:red>depth</text> <text style=color:black>,</text> <text style=color:black>practical</text> <text style=color:black>jump</text> <text style=color:black>drive</text> <text style=color:black>to</text> <text style=color:black>seriously</text> <text style=color:black>conside</text> <text style=color:black>the</text> <text style=color:black>titanium</text> <text style=color:black>cruzer</text>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    }
  ]
}