PayloadEmbeddings_BiLSTM.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "PayloadEmbeddings_BiLSTM",
      "provenance": [],
      "collapsed_sections": [],
      "mount_file_id": "1UF8_SlvYHEKXe1S8xQiYfbyucjlOaMpF",
      "authorship_tag": "ABX9TyP2CVu5EHzy7GV3ScKT+am6",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/SarikaKV/Natural-Language-Processing-and-Text-analysis-Projects/blob/master/PayloadEmbeddings_BiLSTM.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MrS4UiHfi5MO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "from os import path\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from gensim.models import Word2Vec\n",
        "from gensim.models import KeyedVectors\n",
        "from gensim.test.utils import  get_tmpfile\n",
        "from sklearn.model_selection import KFold, cross_validate, GridSearchCV, cross_val_score, StratifiedKFold\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.metrics import make_scorer, accuracy_score\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.ensemble import RandomForestClassifier"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iY2UtKQ3J3ft",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# %tensorflow_version 2.x\n",
        "# import tensorflow as tf\n",
        "# device_name = tf.test.gpu_device_name()\n",
        "# if device_name != '/device:GPU:0':\n",
        "#   raise SystemError('GPU device not found')\n",
        "# print('Found GPU at: {}'.format(device_name))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "izQurgP7jFHH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "input_path = '/content/drive/My Drive/Tozal/data/'\n",
        "dataset = 'CSIC_2010_processed' #'ECML_PKDD_2007_processed' #\n",
        "\n",
        "embedding_dataset = 'CSIC_HTTP_after_sampling.csv' #'ECML_Final_after_sampling.csv' #\n",
        "#emb_sizes = [5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 60, 70, 80, 90, 100]\n",
        "emb_size = 10\n",
        "emb_path = '{}/{}/word2vec_{}.model'.format(input_path, dataset, emb_size)\n",
        "#is_optimize = True\n",
        "pad_strategy='post' \n",
        "trunc_strategy='post'\n",
        "max_length = 500\n",
        "\n",
        "scoring = {'acc': make_scorer(accuracy_score), 'prec': 'precision_micro', \n",
        "           'recall': 'recall_micro', 'f1': 'f1_micro'\n",
        "         }"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "grqi87RLmEkf",
        "colab_type": "code",
        "outputId": "3c547f3f-c611-482f-b2e6-865d01ea432d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "df = pd.read_csv(os.path.join(input_path, dataset, embedding_dataset)) \n",
        "sentences = df.Payload.values\n",
        "df['len'] =  df['Payload'].str.split(\" \").str.len()\n",
        "print(df['len'].mean())\n",
        "num_labels = len(set(df['Label'].values))\n",
        "print(df['Label'].value_counts())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "581.0666467185318\n",
            "1    25065\n",
            "0    25065\n",
            "Name: Label, dtype: int64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WgYorp1xmlf1",
        "colab_type": "code",
        "outputId": "12123ec5-47b7-4909-c4f2-15a3f8a4991f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "# integer encode the documents\n",
        "encoded_docs = [[int(idx) for idx in sentence.split() if idx != '65533'] for sentence in sentences ]\n",
        "\n",
        "# pad documents to a max length of 4 words\n",
        "X = pad_sequences(encoded_docs, maxlen=max_length, padding=pad_strategy, truncating=trunc_strategy)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oW9KvMh2OJn3",
        "colab_type": "code",
        "outputId": "323531d2-33d5-4251-9690-6c43a263b5d4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        }
      },
      "source": [
        "vocab_Set = set()\n",
        "for sent in encoded_docs:\n",
        "  for w in sent:\n",
        "    vocab_Set.add(w)\n",
        "words2id = dict()\n",
        "for i in list(vocab_Set):\n",
        "    words2id[str(i)] = i\n",
        "vocab_size = len(words2id) + 1\n",
        "print(words2id)\n",
        "print(\"Dataset Vocabulary Size: {}\".format(vocab_size))\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'32': 32, '37': 37, '38': 38, '40': 40, '41': 41, '42': 42, '43': 43, '44': 44, '45': 45, '46': 46, '47': 47, '48': 48, '49': 49, '50': 50, '51': 51, '52': 52, '53': 53, '54': 54, '55': 55, '56': 56, '57': 57, '58': 58, '59': 59, '61': 61, '63': 63, '65': 65, '66': 66, '67': 67, '68': 68, '69': 69, '70': 70, '71': 71, '72': 72, '73': 73, '74': 74, '75': 75, '76': 76, '77': 77, '78': 78, '79': 79, '80': 80, '81': 81, '82': 82, '83': 83, '84': 84, '85': 85, '86': 86, '87': 87, '88': 88, '89': 89, '90': 90, '95': 95, '97': 97, '98': 98, '99': 99, '100': 100, '101': 101, '102': 102, '103': 103, '104': 104, '105': 105, '106': 106, '107': 107, '108': 108, '109': 109, '110': 110, '111': 111, '112': 112, '113': 113, '114': 114, '115': 115, '116': 116, '117': 117, '118': 118, '119': 119, '120': 120, '121': 121, '122': 122, '126': 126}\n",
            "Dataset Vocabulary Size: 80\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GZXE78v6jd3M",
        "colab_type": "code",
        "outputId": "c87f67b6-46b7-48ed-b107-e2aaaf82c43f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "# get the embedding matrix from the embedding layer\n",
        "from numpy import zeros\n",
        "w2v = KeyedVectors.load(emb_path, mmap='r')\n",
        "#print(w2v['37'])\n",
        "embedding_matrix = zeros((257, emb_size))\n",
        "print(embedding_matrix.shape)\n",
        "for word, i in words2id.items():\n",
        "  embedding_vector = w2v[str(word)]\n",
        "  if embedding_vector is not None: # and word != '65533': \n",
        "      embedding_matrix[i] = embedding_vector\n",
        "#print(embedding_matrix[32:])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(257, 10)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:253: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
            "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:7: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
            "  import sys\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UWZw4Y4hQNlC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn import preprocessing\n",
        "le = preprocessing.LabelEncoder()\n",
        "Y_new = df['Label']\n",
        "Y_new = le.fit_transform(Y_new)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kFbnCRERTXCA",
        "colab_type": "code",
        "outputId": "0f95b1dd-815b-49ab-f7b7-17c4639ddf4d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "## now splitting into test and training data\n",
        "from sklearn.model_selection import train_test_split\n",
        "X_train,X_test, Y_train, Y_test =  train_test_split(X, Y_new,test_size =0.20,random_state= 4 )\n",
        "print(Y_new)\n",
        "print(set(df['Label'].values))\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[1 1 1 ... 0 0 0]\n",
            "{0, 1}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "li51kKZpQc5l",
        "colab_type": "code",
        "outputId": "ed4c9775-a95c-4011-8bea-311b4eb6ed3a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 575
        }
      },
      "source": [
        "# main model\n",
        "from keras.layers import Input\n",
        "from keras.models import Model\n",
        "from keras.layers.embeddings import Embedding\n",
        "from keras.layers.core import Dense, Reshape, Flatten\n",
        "from keras.layers import dot, TimeDistributed, Bidirectional, LSTM\n",
        "from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n",
        "import time\n",
        "#print(tf.test.gpu_device_name())\n",
        "#with tf.device('/device:GPU:0'):\n",
        "input = Input(shape=(max_length,))\n",
        "model = Embedding(257,emb_size,weights=[embedding_matrix],input_length=max_length)(input)\n",
        "model =  Bidirectional (LSTM (100,return_sequences=True,dropout=0.25),merge_mode='concat')(model)\n",
        "model = TimeDistributed(Dense(100,activation='relu'))(model)\n",
        "model = Flatten()(model)\n",
        "model = Dense(100,activation='relu')(model)\n",
        "if num_labels != 2:\n",
        "  output = Dense(num_labels,activation='softmax')(model)\n",
        "  model = Model(input,output)\n",
        "  model.compile(loss='sparse_categorical_crossentropy',optimizer='adam', metrics=['accuracy'])\n",
        "else:\n",
        "  output = Dense(num_labels-1,activation='sigmoid')(model)\n",
        "  model = Model(input,output)\n",
        "  model.compile(loss='binary_crossentropy',optimizer='adam', metrics=['accuracy'])\n",
        "print(model.summary())\n",
        "\n",
        "earlyStopping = EarlyStopping(monitor='val_loss', patience=3, verbose=0, mode='min')\n",
        "mcp_save = ModelCheckpoint('{}/{}/best_model_es{}_ml{}.hdf5'.format(input_path, dataset, emb_size, max_length), \n",
        "                           save_best_only=True, monitor='val_loss', mode='min', period=1)\n",
        "reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=7, verbose=1, epsilon=1e-4, mode='min')\n",
        "\n",
        "start_train = time.time()\n",
        "history = model.fit(X_train,Y_train,validation_split=0.25, epochs = 20, verbose = 2, callbacks=[earlyStopping, mcp_save,reduce_lr_loss])\n",
        "\n",
        "print(\"Model training took {}mins.\".format((time.time()-start_train)/60))\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_1 (InputLayer)         (None, 500)               0         \n",
            "_________________________________________________________________\n",
            "embedding_1 (Embedding)      (None, 500, 10)           2570      \n",
            "_________________________________________________________________\n",
            "bidirectional_1 (Bidirection (None, 500, 200)          88800     \n",
            "_________________________________________________________________\n",
            "time_distributed_1 (TimeDist (None, 500, 100)          20100     \n",
            "_________________________________________________________________\n",
            "flatten_1 (Flatten)          (None, 50000)             0         \n",
            "_________________________________________________________________\n",
            "dense_2 (Dense)              (None, 100)               5000100   \n",
            "_________________________________________________________________\n",
            "dense_3 (Dense)              (None, 1)                 101       \n",
            "=================================================================\n",
            "Total params: 5,111,671\n",
            "Trainable params: 5,111,671\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/keras/callbacks/callbacks.py:998: UserWarning: `epsilon` argument is deprecated and will be removed, use `min_delta` instead.\n",
            "  warnings.warn('`epsilon` argument is deprecated and '\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/indexed_slices.py:434: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
            "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Train on 30078 samples, validate on 10026 samples\n",
            "Epoch 1/20\n",
            " - 790s - loss: 0.3917 - accuracy: 0.7996 - val_loss: 0.2949 - val_accuracy: 0.8469\n",
            "Epoch 2/20\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W5q6cwbDc9Cs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "print(history.history.keys())\n",
        "# summarize history for accuracy\n",
        "plt.plot(history.history['accuracy'])\n",
        "plt.plot(history.history['val_accuracy'])\n",
        "plt.title('model accuracy')\n",
        "plt.ylabel('accuracy')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train_accuracy', 'validation_accuracy'], loc='upper left')\n",
        "plt.savefig('{}/{}/val_acc_es{}_ml{}.png'.format(input_path, dataset, emb_size, max_length))\n",
        "plt.show()\n",
        "print('{}/{}/val_acc_es{}_ml{}.png'.format(input_path, dataset, emb_size, max_length))\n",
        "# summarize history for loss\n",
        "plt.plot(history.history['loss'])\n",
        "plt.plot(history.history['val_loss'])\n",
        "plt.title('model loss')\n",
        "plt.ylabel('loss')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train_accuracy', 'validation_accuracy','train_loss', 'validation_loss'], loc='upper left')\n",
        "plt.savefig('{}/{}/val_loss_es{}_ml{}.png'.format(input_path, dataset, emb_size, max_length))\n",
        "plt.show()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tAzKswGZac32",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# evaluate the model\n",
        "print('{}/{}/best_model_es{}_ml{}.hdf5'.format(input_path, dataset, emb_size, max_length))\n",
        "model.load_weights('{}/{}/best_model_es{}_ml{}.hdf5'.format(input_path, dataset, emb_size, max_length))\n",
        "loss, accuracy = model.evaluate(X_test, Y_test, verbose=2)\n",
        "print('Accuracy: %f' % (accuracy*100))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tSQSFYGJazyG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.metrics import classification_report,confusion_matrix\n",
        "Y_pred = model.predict(X_test)\n",
        "y_pred = np.array([np.argmax(pred) for pred in Y_pred])\n",
        "print('  Classification Report:\\n',classification_report(Y_test,y_pred),'\\n')"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}