sentiment_analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.3"
    },
    "colab": {
      "name": "sentiment_analysis.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/sayarghoshroy/Intro_to_DL_tutorial/blob/master/sentiment_analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KhGNEf8GxBVD"
      },
      "source": [
        "import json\n",
        "import csv\n",
        "import numpy as np\n",
        "import re\n",
        "import nltk\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import tqdm as tqdm\n",
        "import random\n",
        "import joblib\n",
        "import time\n",
        "from sklearn.neural_network import MLPClassifier\n",
        "from sklearn.datasets import make_classification\n",
        "from sklearn.feature_extraction.text import TfidfTransformer\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.model_selection import KFold"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UEtFLTZ7xBVJ"
      },
      "source": [
        "%%capture .logs\n",
        "# Getting Text Processing Tools\n",
        "\n",
        "nltk.download('all')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lcGR8siZxBVL"
      },
      "source": [
        "# Importing Tools\n",
        "from nltk.tokenize import sent_tokenize, word_tokenize\n",
        "from nltk.tokenize import RegexpTokenizer\n",
        "from string import punctuation\n",
        "from nltk.corpus import stopwords\n",
        "\n",
        "stopword_set = set(stopwords.words('english'))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xKnOwp8fxBVN"
      },
      "source": [
        "with open('train.json', 'r+') as f:\n",
        "    records = json.load(f)\n",
        "\n",
        "with open('test.json', 'r') as f:\n",
        "    gold_test_list = json.load(f)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WRHw7y0T-PX5"
      },
      "source": [
        "X_train = []\n",
        "Y_train = []\n",
        "\n",
        "X_test = []\n",
        "Y_test = []\n",
        "\n",
        "for item in records:\n",
        "  X_train.append(item['content'])\n",
        "  Y_train.append(item['label'])\n",
        "\n",
        "for item in gold_test_list:\n",
        "  X_test.append(item['content'])\n",
        "  Y_test.append(item['label'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZlmjqtLixBVO"
      },
      "source": [
        "def clean(s):\n",
        "    # takes an input string\n",
        "    # preprocesses it for the tf-idf vectorizer\n",
        "    s.replace(\"\\n\", \" \")\n",
        "    tokens = word_tokenize(s)\n",
        "    output = \"\"\n",
        "    \n",
        "    for token in tokens:\n",
        "        unit = token.strip().lower()\n",
        "        if unit in stopword_set or unit in punctuation:\n",
        "            continue\n",
        "        output = output + \" \" + unit\n",
        "        \n",
        "    return output.strip()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RF5MOGLYxBVR"
      },
      "source": [
        "vectorizer = TfidfVectorizer(\n",
        "        sublinear_tf = True,\n",
        "        norm = \"l2\",\n",
        "        encoding = 'utf-8',\n",
        "        max_features = 512,\n",
        "        stop_words = 'english',\n",
        "        ngram_range = (1, 3),\n",
        "        strip_accents = 'unicode',\n",
        "        smooth_idf = True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wJ6h7HVrxBVS",
        "outputId": "60cf17da-35be-4d0b-bd02-3543db265b1a"
      },
      "source": [
        "# To verify correctness of Vectorizer\n",
        "X_train_vec = vectorizer.fit_transform(X_train)\n",
        "print(np.shape(X_train_vec))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(25000, 512)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jV5U_ZclxBVT",
        "outputId": "3328ca6a-6fa0-412c-e26d-297a585b392d"
      },
      "source": [
        "print(\"Size of Train: \" + str(len(X_train)))\n",
        "print(\"Size of Test: \" + str(len(X_test)))\n",
        "max_feature_size = 10000"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Size of Train: 25000\n",
            "Size of Test: 25000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5CXY6994xBVT"
      },
      "source": [
        "def train(X, y, active = 'identity', solve = 'sgd', approach = 'mlp'):\n",
        "    start = time.time()\n",
        "    vec = vectorizer.fit(X)\n",
        "    X_train_vec = vec.transform(X)\n",
        "    \n",
        "    if approach == 'lda':\n",
        "        model = LinearDiscriminantAnalysis()\n",
        "        model.fit(X_train_vec.toarray(), y)\n",
        "    \n",
        "    elif approach == 'mlp':\n",
        "        model = MLPClassifier(alpha = 0,\n",
        "                              hidden_layer_sizes = (512, 1024, 512, 256, 128, 64, 32, 16, 8, 4, 1),\n",
        "                              random_state = 2020,\n",
        "                              activation = active,\n",
        "                              max_iter = int(1e3),\n",
        "                              solver = solve,\n",
        "                              learning_rate = 'adaptive',\n",
        "                              early_stopping = True,\n",
        "                              momentum = 0.9,\n",
        "                              batch_size = 512)\n",
        "        \n",
        "        model.fit(X_train_vec, y)\n",
        "    \n",
        "    end = time.time()\n",
        "    time_to_train = int(round(end - start))\n",
        "\n",
        "    hours = int(time_to_train / 3600)\n",
        "    minutes = int(int(time_to_train % 3600) / 60)\n",
        "    seconds = int(time_to_train % 60)\n",
        "\n",
        "    print()\n",
        "    print('Time taken for training: ' + str(hours).zfill(2) + ':' +\n",
        "          str(minutes).zfill(2) + ':' + str(seconds).zfill(2))\n",
        "    return vec, model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GlUIOFTbxBVW"
      },
      "source": [
        "def get_res(vec, clf):\n",
        "    X_test_vec = vec.transform(X_test)\n",
        "    pred_Y_test = clf.predict(X_test_vec)\n",
        "    print(\"Number of Features: \" + str(np.shape(X_test_vec)[1]))\n",
        "    print(classification_report(Y_test, pred_Y_test, digits = 6))\n",
        "    return"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U6FJ0MyqxBVX",
        "outputId": "ae913549-29c3-4478-bbce-bec3a403a221"
      },
      "source": [
        "# To Try out all possibilities\n",
        "try_all = False\n",
        "\n",
        "if try_all == True:\n",
        "    activations = ['identity', 'tanh', 'relu']\n",
        "    solvers = ['adam', 'sgd', 'lbfgs']\n",
        "else:\n",
        "    activations = ['tanh']\n",
        "    solvers = ['sgd']\n",
        "\n",
        "for active in activations:\n",
        "    for solver in solvers:\n",
        "        if active == 'tanh' and solver == 'lbfgs':\n",
        "            continue\n",
        "        vec, model = train(X_train, Y_train, active, solver)\n",
        "        print(\"Hidden Layer Activation = \" + str(active) + \", Solver = \" + str(solver))\n",
        "        get_res(vec, model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Time taken for training: 00:29:42\n",
            "Hidden Layer Activation = tanh, Solver = sgd\n",
            "Number of Features: 512\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0   0.845930  0.814800  0.830073     12500\n",
            "           1   0.821373  0.851600  0.836214     12500\n",
            "\n",
            "    accuracy                       0.833200     25000\n",
            "   macro avg   0.833652  0.833200  0.833144     25000\n",
            "weighted avg   0.833652  0.833200  0.833144     25000\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i3Cr4A2GxBVX",
        "outputId": "4bd572a8-0b1e-4104-9be8-443398c7a28d"
      },
      "source": [
        "# Testing out a basic pipeline\n",
        "pipe = Pipeline([('Feature Builder', vec), ('Classifier', model)])\n",
        "pred_Y_test = pipe.predict(X_test)\n",
        "print(classification_report(Y_test, pred_Y_test, digits = 6))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0   0.845930  0.814800  0.830073     12500\n",
            "           1   0.821373  0.851600  0.836214     12500\n",
            "\n",
            "    accuracy                       0.833200     25000\n",
            "   macro avg   0.833652  0.833200  0.833144     25000\n",
            "weighted avg   0.833652  0.833200  0.833144     25000\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kFgTyEV3xBVY"
      },
      "source": [
        "# K-fold Cross Validation\n",
        "\n",
        "X = X_train\n",
        "Y = Y_train\n",
        "\n",
        "def cross_val(algo = 'mlp', splits = 5):\n",
        "    global X, Y\n",
        "    splits = int(splits)\n",
        "    if splits > 9 or splits < 3:\n",
        "        splits = 5\n",
        "    print(\"Classification Technique: \" + str(algo))\n",
        "    kf = KFold(n_splits = splits, shuffle = True, random_state = 2020)\n",
        "    index = 1    \n",
        "\n",
        "    for train_index, test_index in kf.split(X):\n",
        "        X_train = []\n",
        "        X_test = []\n",
        "        Y_train = []\n",
        "        Y_test = []\n",
        "\n",
        "        for idx in train_index:\n",
        "            X_train.append(X[idx])\n",
        "            Y_train.append(Y[idx])\n",
        "\n",
        "        for idx in test_index:\n",
        "            X_test.append(X[idx])\n",
        "            Y_test.append(Y[idx])\n",
        "\n",
        "        if algo == 'lda':\n",
        "            vec, model = train(X_train, Y_train, '', '', 'lda')\n",
        "        else:\n",
        "            vec, model = train(X_train, Y_train, 'tanh', 'sgd', 'mlp')\n",
        "\n",
        "        pipe = Pipeline([('Feature Builder', vec), ('Classifier', model)])\n",
        "        pred_Y_test = pipe.predict(X_test)\n",
        "\n",
        "        print(\"Fold Index: \" + str(index))\n",
        "        index += 1\n",
        "        print(classification_report(Y_test, pred_Y_test, digits = 6))\n",
        "        \n",
        "    return"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ILBsC3-pxBVZ",
        "outputId": "319ce6e6-b5ee-4206-d3ad-ef1d1aaee6e3"
      },
      "source": [
        "# Performing K-Fold Cross Validation using LDA\n",
        "cross_val('lda', splits = 3)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Classification Technique: lda\n",
            "\n",
            "Time taken for training: 00:00:39\n",
            "Fold Index: 1\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0   0.838678  0.819029  0.828737      4183\n",
            "           1   0.821840  0.841243  0.831429      4151\n",
            "\n",
            "    accuracy                       0.830094      8334\n",
            "   macro avg   0.830259  0.830136  0.830083      8334\n",
            "weighted avg   0.830292  0.830094  0.830078      8334\n",
            "\n",
            "\n",
            "Time taken for training: 00:00:38\n",
            "Fold Index: 2\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0   0.840903  0.813250  0.826846      4166\n",
            "           1   0.819238  0.846172  0.832487      4167\n",
            "\n",
            "    accuracy                       0.829713      8333\n",
            "   macro avg   0.830071  0.829711  0.829666      8333\n",
            "weighted avg   0.830069  0.829713  0.829667      8333\n",
            "\n",
            "\n",
            "Time taken for training: 00:00:39\n",
            "Fold Index: 3\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0   0.848914  0.809444  0.828709      4151\n",
            "           1   0.819200  0.857006  0.837677      4182\n",
            "\n",
            "    accuracy                       0.833313      8333\n",
            "   macro avg   0.834057  0.833225  0.833193      8333\n",
            "weighted avg   0.834002  0.833313  0.833209      8333\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LovufthcxBVZ",
        "outputId": "b903bb41-41b4-4a1c-d520-485eab216faf"
      },
      "source": [
        "# Performing K-Fold Cross Validation using MLP\n",
        "cross_val('mlp', splits = 3)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Classification Technique: mlp\n",
            "\n",
            "Time taken for training: 00:44:47\n",
            "Fold Index: 1\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0   0.797916  0.787234  0.792539      4183\n",
            "           1   0.788448  0.799085  0.793731      4151\n",
            "\n",
            "    accuracy                       0.793137      8334\n",
            "   macro avg   0.793182  0.793159  0.793135      8334\n",
            "weighted avg   0.793200  0.793137  0.793133      8334\n",
            "\n",
            "\n",
            "Time taken for training: 00:12:53\n",
            "Fold Index: 2\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0   0.740989  0.345415  0.471185      4166\n",
            "           1   0.573306  0.879290  0.694071      4167\n",
            "\n",
            "    accuracy                       0.612384      8333\n",
            "   macro avg   0.657147  0.612352  0.582628      8333\n",
            "weighted avg   0.657137  0.612384  0.582641      8333\n",
            "\n",
            "\n",
            "Time taken for training: 00:11:12\n",
            "Fold Index: 3\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0   0.623016  0.529511  0.572470      4151\n",
            "           1   0.593548  0.681970  0.634695      4182\n",
            "\n",
            "    accuracy                       0.606024      8333\n",
            "   macro avg   0.608282  0.605741  0.603582      8333\n",
            "weighted avg   0.608227  0.606024  0.603698      8333\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HEQxvbr3xBVa",
        "outputId": "6f75d8c3-4c19-46ec-8e5a-87b2744a2fe5"
      },
      "source": [
        "# Training a LDA Classifier on the complete dataset\n",
        "# And saving the full pipeline into a Model\n",
        "\n",
        "vec, model = train(X, Y, '', '', 'lda')\n",
        "\n",
        "pipe = Pipeline([('Feature Builder', vec), ('Classifier', model)])\n",
        "joblib.dump(pipe, \"tf-idf_lda_model.pkl\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Time taken for training: 00:00:57\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['tf-idf_lda_model.pkl']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "00z5qJsExBVa",
        "outputId": "8ddae691-41fa-4bee-bf38-deaf93d74f44"
      },
      "source": [
        "# Training a MLP Classifier on the complete dataset\n",
        "# And saving the full pipeline into a Model\n",
        "\n",
        "vec, model = train(X, Y, 'tanh', 'sgd', 'mlp')\n",
        "\n",
        "pipe = Pipeline([('Feature Builder', vec), ('Classifier', model)])\n",
        "joblib.dump(pipe, \"tf-idf_mlp_model.pkl\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Time taken for training: 00:27:22\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['tf-idf_mlp_model.pkl']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e8o40jQYxBVa",
        "outputId": "c9a380c4-7bcc-4355-ae8b-ecf924ca2d97"
      },
      "source": [
        "# Testing out the saved pipeline on all train samples\n",
        "saved_pipe = joblib.load(\"tf-idf_lda_model.pkl\")\n",
        "\n",
        "pred_Y_all = saved_pipe.predict(X)\n",
        "print(classification_report(Y, pred_Y_all, digits = 6))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0   0.854967  0.826240  0.840358     12500\n",
            "           1   0.831889  0.859840  0.845633     12500\n",
            "\n",
            "    accuracy                       0.843040     25000\n",
            "   macro avg   0.843428  0.843040  0.842996     25000\n",
            "weighted avg   0.843428  0.843040  0.842996     25000\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4WzlKu7gxBVa",
        "outputId": "33bfdd30-cadb-4f1b-c7f0-4c1cce82b05c"
      },
      "source": [
        "# Testing out Saved LDA Model on Test Data\n",
        "\n",
        "saved_pipe = joblib.load(\"tf-idf_lda_model.pkl\")\n",
        "\n",
        "X_gold_test = []\n",
        "Y_gold_test = []\n",
        "\n",
        "for unit in gold_test_list:\n",
        "    X_gold_test.append(unit['content'])\n",
        "    Y_gold_test.append(unit['label'])\n",
        "    \n",
        "pred_Y_gold_test = saved_pipe.predict(X_gold_test)\n",
        "print(classification_report(Y_gold_test, pred_Y_gold_test, digits = 6))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0   0.843693  0.816560  0.829905     12500\n",
            "           1   0.822276  0.848720  0.835289     12500\n",
            "\n",
            "    accuracy                       0.832640     25000\n",
            "   macro avg   0.832984  0.832640  0.832597     25000\n",
            "weighted avg   0.832984  0.832640  0.832597     25000\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YdLrWo0hxBVa",
        "outputId": "718d7644-d215-4efb-9f02-5c595a2eea79"
      },
      "source": [
        "# Testing out Saved MLP Model on Test Data\n",
        "\n",
        "saved_pipe = joblib.load(\"tf-idf_mlp_model.pkl\")\n",
        "\n",
        "X_gold_test = []\n",
        "Y_gold_test = []\n",
        "\n",
        "for unit in gold_test_list:\n",
        "    X_gold_test.append(unit['content'])\n",
        "    Y_gold_test.append(unit['label'])\n",
        "    \n",
        "pred_Y_gold_test = saved_pipe.predict(X_gold_test)\n",
        "print(classification_report(Y_gold_test, pred_Y_gold_test, digits = 6))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0   0.845930  0.814800  0.830073     12500\n",
            "           1   0.821373  0.851600  0.836214     12500\n",
            "\n",
            "    accuracy                       0.833200     25000\n",
            "   macro avg   0.833652  0.833200  0.833144     25000\n",
            "weighted avg   0.833652  0.833200  0.833144     25000\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "adDYwu0OM94m"
      },
      "source": [
        "### $Exercise$:\n",
        "\n",
        "#### $Replace\\ TF-IDF\\ with\\ GloVe\\ Vectors,\\ re-run\\ experiments.$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PEt9QxQDxBVf"
      },
      "source": [
        "# ^_^ Thank You"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}