nlp_political_bias.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.7"
    },
    "colab": {
      "name": "Next Word Prediction-1.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/scaperex/My_Projects/blob/master/nlp_political_bias.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LNUFrq7lqHSg"
      },
      "source": [
        "### Importing The Required Libraries:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f151CllrqHSj"
      },
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "from tensorflow.keras.layers import Embedding, LSTM, Dense\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.utils import to_categorical\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "import pickle\n",
        "import numpy as np\n",
        "import os\n",
        "import string\n",
        "from tensorflow import keras\n",
        "from keras.utils.vis_utils import plot_model\n",
        "from tensorflow.keras.callbacks import ModelCheckpoint\n",
        "from tensorflow.keras.callbacks import ReduceLROnPlateau\n",
        "from tensorflow.keras.callbacks import TensorBoard"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "94f4WitEtTbB"
      },
      "source": [
        "# Mount Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mSstXrpJqQ0r",
        "outputId": "391676d0-aaf3-4f52-a413-5ac2495c0fab",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# from google.colab import drive\n",
        "# drive.mount('/content/drive')"
      ],
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "07-tIa63qHSe"
      },
      "source": [
        "# Next Word Prediction:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JHcstz938J2f",
        "outputId": "fa59e113-5550-472a-cc84-3e12a8c3c712",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "def preprocess(base_path = '/content/drive/My Drive/NewB-master', file_name='example.txt'):\n",
        "    \n",
        "    data = \"\"\n",
        "\n",
        "    with open(os.path.join(base_path,file_name), 'r') as f:\n",
        "        for line in f:\n",
        "            line = line.strip()\n",
        "            label, sentence = line.split('\\t')[0:2] # remove label and \\n\n",
        "            if label == '1': # 1 - New York Times {Liberal}, 7 - New York Post {Conservative}\n",
        "                data += sentence\n",
        "            # data += ' *start* '\n",
        "            \n",
        "    \n",
        "    # Tokenization\n",
        "    tokenizer = Tokenizer()\n",
        "    tokenizer.fit_on_texts([data])\n",
        "\n",
        "    # saving the tokenizer for predict function.\n",
        "    pickle.dump(tokenizer, open(f'tokenizer_{file_name.split(\".\")[0]}.pkl', 'wb'))\n",
        "\n",
        "    sequence_data = tokenizer.texts_to_sequences([data])[0]\n",
        "\n",
        "    vocab_size = len(tokenizer.word_index) + 1\n",
        "    print('vocab_size: ',vocab_size)\n",
        "\n",
        "    sequences = []\n",
        "    for i in range(1, len(sequence_data)):\n",
        "        words = sequence_data[i-1:i+1]\n",
        "        sequences.append(words)\n",
        "        \n",
        "    print(\"The Length of sequences are: \", len(sequences))\n",
        "    sequences = np.array(sequences)\n",
        "    return  sequences[:,0], sequences[:,1], vocab_size\n",
        "\n",
        "X,y, vocab_size = preprocess(base_path='', file_name='train_orig.txt')"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "vocab_size:  34676\n",
            "The Length of sequences are:  392778\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F-dsSXIvUm3t",
        "outputId": "bf2611b1-ec42-4281-bcce-2b0fdb95c039",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "# data, vocab_size = preprocess()"
      ],
      "execution_count": 131,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "vocab_size:  113263\n",
            "The Length of sequences are:  2547197\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rkrsxL5FqHUr"
      },
      "source": [
        "### Creating the Model:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mRsWCkwbqHUt",
        "outputId": "23758cf9-1192-4bd5-81c5-d1059cfc1364",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 306
        }
      },
      "source": [
        "class nwp_model():\n",
        "    def __init__(self,model=None, embedding_dim=10):\n",
        "        if model:\n",
        "            self.model = model\n",
        "        else:\n",
        "            model = Sequential()\n",
        "            model.add(Embedding(vocab_size, output_dim=embedding_dim,input_length=1))\n",
        "            model.add(LSTM(50))\n",
        "            model.add(Dense(50, activation=\"relu\"))\n",
        "            model.add(Dense(vocab_size, activation=\"softmax\"))\n",
        "            self.model=model\n",
        "        print(model.summary())\n",
        "\n",
        "    # def load_and_process_data(self:)\n",
        "\n",
        "    def train(self, feature, target, num_epochs=50):\n",
        "        checkpoint = ModelCheckpoint(\"nextword1.h5\", monitor='loss', verbose=1, save_best_only=True, mode='auto')\n",
        "\n",
        "        # reduce_LR = ReduceLROnPlateau(monitor='loss', factor=0.2, patience=10, min_lr=0.0001, verbose = 1)\n",
        "\n",
        "        # logdir='logsnextword1'\n",
        "        # tensorboard_Visualization = TensorBoard(log_dir=logdir)\n",
        "        self.model.compile(optimizer=Adam(lr=0.001), loss='SparseCategoricalCrossentropy')\n",
        "        self.model.fit(feature, target, epochs=num_epochs, batch_size=64, callbacks=[checkpoint])#, reduce_LR, tensorboard_Visualization])\n",
        "\n",
        "classifier = nwp_model(embedding_dim=10)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding (Embedding)        (None, 1, 10)             346760    \n",
            "_________________________________________________________________\n",
            "lstm (LSTM)                  (None, 50)                12200     \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 50)                2550      \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 34676)             1768476   \n",
            "=================================================================\n",
            "Total params: 2,129,986\n",
            "Trainable params: 2,129,986\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PRyX-bJ0ZA61",
        "outputId": "d6bae5e6-7f58-4937-ee3b-e00842a5ec21",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 663
        }
      },
      "source": [
        "classifier.train(X,y, num_epochs=10)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/10\n",
            "6138/6138 [==============================] - ETA: 0s - loss: 7.3498\n",
            "Epoch 00001: loss improved from inf to 7.34979, saving model to nextword1.h5\n",
            "6138/6138 [==============================] - 43s 7ms/step - loss: 7.3498\n",
            "Epoch 2/10\n",
            "6133/6138 [============================>.] - ETA: 0s - loss: 6.8338\n",
            "Epoch 00002: loss improved from 7.34979 to 6.83392, saving model to nextword1.h5\n",
            "6138/6138 [==============================] - 45s 7ms/step - loss: 6.8339\n",
            "Epoch 3/10\n",
            "6137/6138 [============================>.] - ETA: 0s - loss: 6.5212\n",
            "Epoch 00003: loss improved from 6.83392 to 6.52115, saving model to nextword1.h5\n",
            "6138/6138 [==============================] - 46s 7ms/step - loss: 6.5211\n",
            "Epoch 4/10\n",
            "6135/6138 [============================>.] - ETA: 0s - loss: 6.2832\n",
            "Epoch 00004: loss improved from 6.52115 to 6.28298, saving model to nextword1.h5\n",
            "6138/6138 [==============================] - 46s 7ms/step - loss: 6.2830\n",
            "Epoch 5/10\n",
            "6133/6138 [============================>.] - ETA: 0s - loss: 6.0849\n",
            "Epoch 00005: loss improved from 6.28298 to 6.08493, saving model to nextword1.h5\n",
            "6138/6138 [==============================] - 46s 7ms/step - loss: 6.0849\n",
            "Epoch 6/10\n",
            "6136/6138 [============================>.] - ETA: 0s - loss: 5.9230\n",
            "Epoch 00006: loss improved from 6.08493 to 5.92296, saving model to nextword1.h5\n",
            "6138/6138 [==============================] - 46s 8ms/step - loss: 5.9230\n",
            "Epoch 7/10\n",
            "6134/6138 [============================>.] - ETA: 0s - loss: 5.7941\n",
            "Epoch 00007: loss improved from 5.92296 to 5.79419, saving model to nextword1.h5\n",
            "6138/6138 [==============================] - 46s 7ms/step - loss: 5.7942\n",
            "Epoch 8/10\n",
            "6132/6138 [============================>.] - ETA: 0s - loss: 5.6901\n",
            "Epoch 00008: loss improved from 5.79419 to 5.69008, saving model to nextword1.h5\n",
            "6138/6138 [==============================] - 45s 7ms/step - loss: 5.6901\n",
            "Epoch 9/10\n",
            "6138/6138 [==============================] - ETA: 0s - loss: 5.6038\n",
            "Epoch 00009: loss improved from 5.69008 to 5.60384, saving model to nextword1.h5\n",
            "6138/6138 [==============================] - 45s 7ms/step - loss: 5.6038\n",
            "Epoch 10/10\n",
            "2174/6138 [=========>....................] - ETA: 30s - loss: 5.5080"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Pa1x-21vqHVB"
      },
      "source": [
        "### Plot The Model:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ouU2ao7DqHVD"
      },
      "source": [
        "keras.utils.plot_model(model, to_file='model.png', show_layer_names=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XWyx7TDRwFea"
      },
      "source": [
        "# Load the TensorBoard notebook extension\n",
        "%load_ext tensorboard\n",
        "%tensorboard --logdir=\"./logsnextword1\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UHAVhHe-qHV-"
      },
      "source": [
        "## Observation:\n",
        "### We are able to develop a decent next word prediction model and are able to get a declining loss and an overall decent performance."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iEEjeyhdu595"
      },
      "source": [
        "# Importing the Libraries\n",
        "\n",
        "from tensorflow.keras.models import load_model\n",
        "import numpy as np\n",
        "import pickle\n",
        "\n",
        "# Load the model and tokenizer\n",
        "\n",
        "model = load_model('nextword1.h5')\n",
        "tokenizer = pickle.load(open('tokenizer_train_orig.pkl', 'rb'))\n",
        "\n",
        "def Predict_Next_Words(model, tokenizer, text):\n",
        "    \"\"\"\n",
        "        In this function we are using the tokenizer and models trained\n",
        "        and we are creating the sequence of the text entered and then\n",
        "        using our model to predict and return the the predicted word.\n",
        "    \n",
        "    \"\"\"\n",
        "    for i in range(3):\n",
        "        sequence = tokenizer.texts_to_sequences([text])[0]\n",
        "        sequence = np.array(sequence)\n",
        "        preds = np.argmax(model.predict(sequence), axis=-1)\n",
        "        # preds = model.predict_classes(sequence)\n",
        "#         print(preds)\n",
        "        predicted_word = \"\"\n",
        "        \n",
        "        for key, value in tokenizer.word_index.items():\n",
        "            if value == preds:\n",
        "                predicted_word = key\n",
        "                break\n",
        "        \n",
        "        print(predicted_word)\n",
        "        return(predicted_word)"
      ],
      "execution_count": 147,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RzRssTdpu711",
        "outputId": "2060008e-1446-4be6-b270-911ae5b9cc0b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 731
        }
      },
      "source": [
        "\"\"\"\n",
        "    We are testing our model and we will run the model\n",
        "    until the user decides to stop the script.\n",
        "    While the script is running we try and check if \n",
        "    the prediction can be made on the text. If no\n",
        "    prediction can be made we just continue.\n",
        "\n",
        "\"\"\"\n",
        "\n",
        "while(True):\n",
        "\n",
        "    text = input(\"Enter your line: \")\n",
        "    \n",
        "    if text == \"x\":\n",
        "        print(\"Ending The Program.....\")\n",
        "        break\n",
        "    \n",
        "    else:\n",
        "        text = text.split(\" \")\n",
        "        text = text[-1]\n",
        "\n",
        "        text = ''.join(text)\n",
        "        for _ in range(9):\n",
        "            text = Predict_Next_Words(model, tokenizer, text)\n",
        "        "
      ],
      "execution_count": 149,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Enter your line: hillary\n",
            "clinton\n",
            "and\n",
            "the\n",
            "trump\n",
            "and\n",
            "the\n",
            "trump\n",
            "and\n",
            "the\n",
            "Enter your line: trump\n",
            "and\n",
            "the\n",
            "trump\n",
            "and\n",
            "the\n",
            "trump\n",
            "and\n",
            "the\n",
            "trump\n",
            "Enter your line: is\n",
            "a\n",
            "trump\n",
            "and\n",
            "the\n",
            "trump\n",
            "and\n",
            "the\n",
            "trump\n",
            "and\n",
            "Enter your line: donald\n",
            "trump\n",
            "and\n",
            "the\n",
            "trump\n",
            "and\n",
            "the\n",
            "trump\n",
            "and\n",
            "the\n",
            "Enter your line: x\n",
            "Ending The Program.....\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}