final.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.6"
    },
    "colab": {
      "name": "final.ipynb",
      "provenance": [],
      "include_colab_link": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Riturao/-Time-TCS-CodeVita-2019-zone-2-/blob/master/final.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OzzUMYkS3efi",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        },
        "outputId": "cd59ef3a-34a5-4bbf-e849-4762e8a8e666"
      },
      "source": [
        "echo \"# fyp\" >> README.md\n",
        "git init\n",
        "git add README.md\n",
        "git commit -m \"first commit\"\n",
        "git remote add origin https://github.com/Riturao/fyp.git\n",
        "git push -u origin master"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "error",
          "ename": "SyntaxError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-52-12e759937b04>\"\u001b[0;36m, line \u001b[0;32m1\u001b[0m\n\u001b[0;31m    echo \"# fyp\" >> README.md\u001b[0m\n\u001b[0m               ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
          ]
        },
        {
          "output_type": "stream",
          "text": [
            "ERROR! Session/line number was not unique in database. History logging moved to new session 59\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "iiA5UUqZKmWj",
        "outputId": "5bc53b4b-bc29-40ca-91db-5c257d3c23ed",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 710
        }
      },
      "source": [
        "pip install tensorflow==1.14"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting tensorflow==1.14\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/de/f0/96fb2e0412ae9692dbf400e5b04432885f677ad6241c088ccc5fe7724d69/tensorflow-1.14.0-cp36-cp36m-manylinux1_x86_64.whl (109.2MB)\n",
            "\u001b[K     |████████████████████████████████| 109.2MB 49kB/s \n",
            "\u001b[?25hCollecting tensorboard<1.15.0,>=1.14.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/91/2d/2ed263449a078cd9c8a9ba50ebd50123adf1f8cfbea1492f9084169b89d9/tensorboard-1.14.0-py3-none-any.whl (3.1MB)\n",
            "\u001b[K     |████████████████████████████████| 3.2MB 33.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (3.10.0)\n",
            "Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (0.3.3)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (0.34.2)\n",
            "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (0.8.1)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (1.27.2)\n",
            "Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (0.2.0)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (1.12.0)\n",
            "Collecting tensorflow-estimator<1.15.0rc0,>=1.14.0rc0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/3c/d5/21860a5b11caf0678fbc8319341b0ae21a07156911132e0e71bffed0510d/tensorflow_estimator-1.14.0-py2.py3-none-any.whl (488kB)\n",
            "\u001b[K     |████████████████████████████████| 491kB 48.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy<2.0,>=1.14.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (1.18.2)\n",
            "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (0.9.0)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (1.1.0)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (1.12.1)\n",
            "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (1.1.0)\n",
            "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (1.0.8)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow==1.14) (46.1.3)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow==1.14) (1.0.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow==1.14) (3.2.1)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow==1.14) (2.10.0)\n",
            "Installing collected packages: tensorboard, tensorflow-estimator, tensorflow\n",
            "  Found existing installation: tensorboard 2.2.0\n",
            "    Uninstalling tensorboard-2.2.0:\n",
            "      Successfully uninstalled tensorboard-2.2.0\n",
            "  Found existing installation: tensorflow-estimator 2.2.0rc0\n",
            "    Uninstalling tensorflow-estimator-2.2.0rc0:\n",
            "      Successfully uninstalled tensorflow-estimator-2.2.0rc0\n",
            "  Found existing installation: tensorflow 2.2.0rc2\n",
            "    Uninstalling tensorflow-2.2.0rc2:\n",
            "      Successfully uninstalled tensorflow-2.2.0rc2\n",
            "Successfully installed tensorboard-1.14.0 tensorflow-1.14.0 tensorflow-estimator-1.14.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9BzdfMWH4rty",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%matplotlib inline\n",
        "import matplotlib.pyplot as plt\n",
        "#import ost as mi\n",
        "import re\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "#from chart_studio import plotly as py\n",
        "#import plotly.figure_factory as ff\n",
        "from scipy import stats\n",
        "\n",
        "import gensim\n",
        "import json\n",
        "\n",
        "\n",
        "import nltk\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk.stem import PorterStemmer\n",
        "\n",
        "import itertools\n",
        "\n",
        "import keras\n",
        "\n",
        "from keras.models import Sequential, Model\n",
        "from keras.layers import Dense, Dropout, Embedding, LSTM, Input, Activation, GlobalAveragePooling1D, Flatten, Concatenate, Conv1D, MaxPooling1D\n",
        "from keras.layers.normalization import BatchNormalization\n",
        "from keras.layers.merge import concatenate\n",
        "from keras.optimizers import SGD, RMSprop, Adagrad, Adam\n",
        "from keras.preprocessing.text import one_hot, text_to_word_sequence, Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
        "from keras.utils.vis_utils import plot_model\n",
        "\n",
        "import fnmatch\n",
        "\n",
        "import warnings\n",
        "\n",
        "import string\n",
        "from pathlib import Path\n",
        "from random import shuffle\n",
        "from ast import literal_eval\n",
        "\n",
        "warnings.filterwarnings('ignore')\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qRDtXmDG4rt5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "wordnet_lemmatizer = WordNetLemmatizer()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "83Hom6vq4rt8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "WINDOWS_SIZE = 10\n",
        "labels=['none','mild','moderate','moderately severe', 'severe']\n",
        "num_classes = len(labels)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J_EtyrZY4rt_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def plot_acc(history, title=\"Model Accuracy\"):\n",
        "    plt.plot(history.history['accuracy'])\n",
        "    plt.plot(history.history['val_accuracy'])\n",
        "    plt.title(title)\n",
        "    plt.ylabel('Accuracy')\n",
        "    plt.xlabel('Epoch')\n",
        "    plt.legend(['Train', 'Val'], loc='upper left')\n",
        "    plt.show()\n",
        "    \n",
        "def plot_loss(history, title=\"Model Loss\"):\n",
        "    plt.plot(history.history['loss'])\n",
        "    plt.plot(history.history['val_loss'])\n",
        "    plt.title(title)\n",
        "    plt.ylabel('Loss')\n",
        "    plt.xlabel('Epoch')\n",
        "    plt.legend(['Train', 'Val'], loc='upper right')\n",
        "    plt.show()\n",
        "    \n",
        "def plot_compare_losses(history1, history2, name1=\"Red 1\", name2=\"Red 2\", title=\"Graph title\"):\n",
        "    plt.plot(history1.history['loss'], color=\"green\")\n",
        "    plt.plot(history1.history['val_loss'], 'r--', color=\"green\")\n",
        "    plt.plot(history2.history['loss'], color=\"blue\")\n",
        "    plt.plot(history2.history['val_loss'], 'r--', color=\"blue\")\n",
        "    plt.title(title)\n",
        "    plt.ylabel('Loss')\n",
        "    plt.xlabel('Epoch')\n",
        "    plt.legend(['Train ' + name1, 'Val ' + name1, \n",
        "                'Train ' + name2, 'Val ' + name2],\n",
        "               loc='upper right')\n",
        "    plt.show()\n",
        "    \n",
        "def plot_compare_accs(history1, history2, name1=\"Red 1\",\n",
        "                      name2=\"Red 2\", title=\"Graph title\"):\n",
        "    \"\"\"Compara accuracies de dos entrenamientos con nombres name1 y name2\"\"\"\n",
        "    plt.plot(history1.history['acc'], color=\"green\")\n",
        "    plt.plot(history1.history['val_acc'], 'r--', color=\"green\")\n",
        "    plt.plot(history2.history['acc'], color=\"blue\")\n",
        "    plt.plot(history2.history['val_acc'], 'r--', color=\"blue\")\n",
        "    plt.title(title)\n",
        "    plt.ylabel('Accuracy')\n",
        "    plt.xlabel('Epoch')\n",
        "    plt.legend(['Train ' + name1, 'Val ' + name1, \n",
        "                'Train ' + name2, 'Val ' + name2], \n",
        "               loc='lower right')\n",
        "    plt.show()\n",
        "\n",
        "def plot_compare_multiple_metrics(history_array, names, colors, title=\"Graph title\", metric='acc'):  \n",
        "    legend = []\n",
        "    for i in range(0, len(history_array)):\n",
        "        plt.plot(history_array[i].history[metric], color=colors[i])\n",
        "        plt.plot(history_array[i].history['val_' + metric], 'r--', color=colors[i])\n",
        "        legend.append('Train ' + names[i])\n",
        "        legend.append('Val ' + names[i])\n",
        "    \n",
        "    plt.title(title)\n",
        "    plt.ylabel('Accuracy')\n",
        "    plt.xlabel('Epoch')   \n",
        "    plt.axis\n",
        "    plt.legend(legend, \n",
        "               loc='lower right')\n",
        "    plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6qVQhbGR4ruD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def transcripts_to_dataframe(directory):\n",
        "    rows_list = []\n",
        "        \n",
        "    filenames = os.listdir(directory)\n",
        "    \n",
        "    if \".DS_Store\" in filenames:\n",
        "        filenames.remove(\".DS_Store\")\n",
        "        \n",
        "    for filename in filenames:\n",
        "        \n",
        "        transcript_path = os.path.join(directory, filename)\n",
        "        transcript = pd.read_csv(transcript_path , sep='\\t')\n",
        "        #print(transcript)\n",
        "        m = re.search(\"(\\d{3})_TRANSCRIPT.csv\", filename)\n",
        "        if m:\n",
        "            print(filename)\n",
        "            person_id = m.group(1)\n",
        "            #print(person_id)\n",
        "            p = {}\n",
        "            question = \"\"\n",
        "            answer = \"\"\n",
        "            lines = len(transcript)\n",
        "            #print(lines)\n",
        "            for i in range(0, lines):\n",
        "                #print(\"entered\")\n",
        "                row = transcript.iloc[i]\n",
        "                #print(row)\n",
        "                if (row[\"speaker\"] == \"Ellie\") or (i == lines - 1):\n",
        "                    p[\"personId\"] = person_id\n",
        "                    #print(p)\n",
        "                    if \"(\" in str(question):\n",
        "                        question = question[question.index(\"(\") + 1:question.index(\")\")]\n",
        "                    p[\"question\"] = question\n",
        "                    p[\"answer\"] = answer\n",
        "                    #print(answer)\n",
        "                    #print(p)\n",
        "                    if question != \"\":\n",
        "                        rows_list.append(p)\n",
        "                        #print(rows_list())\n",
        "                    p = {}\n",
        "                    answer = \"\"\n",
        "                    question = row[\"value\"]\n",
        "                else:\n",
        "                    answer = str(answer) + \" \" + str(row[\"value\"])\n",
        "                    #print(1)\n",
        "                #print(answer)\n",
        "\n",
        "    all_participants = pd.DataFrame(rows_list, columns=['personId', 'question', 'answer'])\n",
        "    #all_participants = pd.DataFrame(*p, columns=['personId', 'question', 'answer'])\n",
        "    all_participants.to_csv(directory + 'all/all.csv', sep=',')\n",
        "    print(\"File was created\")\n",
        "    return all_participants"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R1GO9hzo6K2k",
        "colab_type": "code",
        "outputId": "51e37d13-4a9b-436e-e28f-e29d359a6c3b",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 181
        }
      },
      "source": [
        "from google.colab import files\n",
        "uploaded = files.upload()"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-d3c3a5db-04de-4028-b498-4d06a4ed6719\" name=\"files[]\" multiple disabled />\n",
              "     <output id=\"result-d3c3a5db-04de-4028-b498-4d06a4ed6719\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving all.csv to all (1).csv\n",
            "Saving ds_total.csv to ds_total (1).csv\n",
            "Saving phrases_lp.csv to phrases_lp (1).csv\n",
            "Saving phrases_lp_stop.csv to phrases_lp_stop (1).csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AUCWdHKX6NkV",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 90
        },
        "outputId": "48d1346a-561c-4f24-ecf9-9d77249a8aa5"
      },
      "source": [
        "for fn in uploaded.keys():\n",
        "  print('User uploaded file \"{name}\" with length {length} bytes'.format(\n",
        "      name=fn, length=len(uploaded[fn])))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "User uploaded file \"all.csv\" with length 2007054 bytes\n",
            "User uploaded file \"ds_total.csv\" with length 7638 bytes\n",
            "User uploaded file \"phrases_lp.csv\" with length 20903991 bytes\n",
            "User uploaded file \"phrases_lp_stop.csv\" with length 37898874 bytes\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fE1iDg25SqtO",
        "colab_type": "code",
        "outputId": "597a07b2-279e-4806-c2de-ff7d0832cda5",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 39
        }
      },
      "source": [
        "from google.colab import files\n",
        "uploaded = files.upload()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-8221b5f6-883a-41aa-b7c1-78663654b4fa\" name=\"files[]\" multiple disabled />\n",
              "     <output id=\"result-8221b5f6-883a-41aa-b7c1-78663654b4fa\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kv83jngCWlE_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for fn in uploaded.keys():\n",
        "  print('User uploaded file \"{name}\" with length {length} bytes'.format(\n",
        "      name=fn, length=len(uploaded[fn])))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kWi7OC-_WzNC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4RDjLWFoWpuF",
        "colab_type": "code",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 145
        },
        "outputId": "c2b8c6a2-d9f6-421d-ec2a-34d5a50ff496"
      },
      "source": [
        "from google.colab import files\n",
        "uploaded = files.upload()"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-a2e16ec1-efaa-4606-8d49-7ade056f7a5b\" name=\"files[]\" multiple disabled />\n",
              "     <output id=\"result-a2e16ec1-efaa-4606-8d49-7ade056f7a5b\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving dev_split_Depression_AVEC2017.csv to dev_split_Depression_AVEC2017.csv\n",
            "Saving full_test_split.csv to full_test_split.csv\n",
            "Saving train_split_Depression_AVEC2017.csv to train_split_Depression_AVEC2017.csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NplV87zCW0eV",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "outputId": "bc33de05-bb53-458a-fec6-de46b3718c5a"
      },
      "source": [
        "for fn in uploaded.keys():\n",
        "  print('User uploaded file \"{name}\" with length {length} bytes'.format(\n",
        "      name=fn, length=len(uploaded[fn])))"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "User uploaded file \"dev_split_Depression_AVEC2017.csv\" with length 1112 bytes\n",
            "User uploaded file \"full_test_split.csv\" with length 620 bytes\n",
            "User uploaded file \"train_split_Depression_AVEC2017.csv\" with length 3074 bytes\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b8GZ6FGzW6Zl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/gdrive')\n",
        "%cd /gdrive"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OVPU2JeHXHBk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9rvLXpCu4ruH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#data_path = r\"C:/Users/Admin/Desktop/transcripts/\"\n",
        "#transcripts_to_dataframe(data_path) \n",
        "all_participants = pd.read_csv( 'all.csv', sep=',')\n",
        "#all_participants= pd.read_csv(data_path + '*_TRANSCRIPT.csv', sep=',')\n",
        "#all_participants = pd.concat([pd.read_csv(f) for f in glob.glob('*_TRANSCRIPT.csv')], ignore_index = True)#"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "GyL_uORf4ruL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 198
        },
        "outputId": "765c044f-09ab-4bbc-ac85-a71b64553f5c"
      },
      "source": [
        "all_participants.tail() "
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>personId</th>\n",
              "      <th>question</th>\n",
              "      <th>answer</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>14968</th>\n",
              "      <td>14968</td>\n",
              "      <td>492</td>\n",
              "      <td>mm</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14969</th>\n",
              "      <td>14969</td>\n",
              "      <td>492</td>\n",
              "      <td>what are you most proud of in your life</td>\n",
              "      <td>um i am very proud of the fact that uh i don'...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14970</th>\n",
              "      <td>14970</td>\n",
              "      <td>492</td>\n",
              "      <td>okay i think i have asked everything i need to</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14971</th>\n",
              "      <td>14971</td>\n",
              "      <td>492</td>\n",
              "      <td>thanks for sharing your thoughts with me</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14972</th>\n",
              "      <td>14972</td>\n",
              "      <td>492</td>\n",
              "      <td>goodbye</td>\n",
              "      <td>bye and thank you</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       Unnamed: 0  ...                                             answer\n",
              "14968       14968  ...                                                NaN\n",
              "14969       14969  ...   um i am very proud of the fact that uh i don'...\n",
              "14970       14970  ...                                                NaN\n",
              "14971       14971  ...                                                NaN\n",
              "14972       14972  ...                                  bye and thank you\n",
              "\n",
              "[5 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PuguhdBh4ruP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "all_participants.columns =  ['index','personId', 'question', 'answer']\n",
        "all_participants = all_participants.astype({\"index\": int, \"personId\": float, \"question\": str, \"answer\": str })"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mmh0jdq34ruS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#all_participants = pd.DataFrame(columns=['index','personId', 'question', 'answer'])\n",
        "all_participants.columns =  ['index','personId', 'question', 'answer']\n",
        "all_participants = all_participants.astype({\"index\": int, \"personId\": float, \"question\": str, \"answer\": str })"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F6_giEEB4ruV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def text_to_wordlist(text, remove_stopwords=True, stem_words=False):    \n",
        "    # Clean the text, with the option to remove stopwords and to stem words.\n",
        "    \n",
        "    # Convert words to lower case and split them\n",
        "    text = text.lower().split()\n",
        "\n",
        "    # Optionally, remove stop words\n",
        "    if remove_stopwords:\n",
        "        stops = set(stopwords.words(\"english\"))\n",
        "        text = [wordnet_lemmatizer.lemmatize(w) for w in text if not w in stops ]\n",
        "        text = [w for w in text if w != \"nan\" ]\n",
        "    else:\n",
        "        text = [wordnet_lemmatizer.lemmatize(w) for w in text]\n",
        "        text = [w for w in text if w != \"nan\" ]\n",
        "    \n",
        "    text = \" \".join(text)\n",
        "\n",
        "    # Clean the text\n",
        "    text = re.sub(r\"[^A-Za-z0-9^,!.\\/'+-=]\", \" \", text)\n",
        "    text = re.sub(r\"what's\", \"what is \", text)\n",
        "    text = re.sub(r\"\\'s\", \" \", text)\n",
        "    text = re.sub(r\"\\'ve\", \" have \", text)\n",
        "    text = re.sub(r\"can't\", \"cannot \", text)\n",
        "    text = re.sub(r\"n't\", \" not \", text)\n",
        "    text = re.sub(r\"i'm\", \"i am \", text)\n",
        "    text = re.sub(r\"\\'re\", \" are \", text)\n",
        "    text = re.sub(r\"\\'d\", \" would \", text)\n",
        "    text = re.sub(r\"\\'ll\", \" will \", text)\n",
        "    text = re.sub(r\",\", \" \", text)\n",
        "    text = re.sub(r\"\\.\", \" \", text)\n",
        "    text = re.sub(r\"!\", \" ! \", text)\n",
        "    text = re.sub(r\"\\/\", \" \", text)\n",
        "    text = re.sub(r\"\\^\", \" ^ \", text)\n",
        "    text = re.sub(r\"\\+\", \" + \", text)\n",
        "    text = re.sub(r\"\\-\", \" - \", text)\n",
        "    text = re.sub(r\"\\=\", \" = \", text)\n",
        "    \n",
        "    text = re.sub(r\"\\<\", \" \", text)\n",
        "    text = re.sub(r\"\\>\", \" \", text)\n",
        "    \n",
        "    text = re.sub(r\"'\", \" \", text)\n",
        "    text = re.sub(r\"(\\d+)(k)\", r\"\\g<1>000\", text)\n",
        "    text = re.sub(r\":\", \" : \", text)\n",
        "    text = re.sub(r\" e g \", \" eg \", text)\n",
        "    text = re.sub(r\" b g \", \" bg \", text)\n",
        "    text = re.sub(r\" u s \", \" american \", text)\n",
        "    text = re.sub(r\"\\0s\", \"0\", text)\n",
        "    text = re.sub(r\" 9 11 \", \"911\", text)\n",
        "    text = re.sub(r\"e - mail\", \"email\", text)\n",
        "    text = re.sub(r\"j k\", \"jk\", text)\n",
        "    text = re.sub(r\"\\s{2,}\", \" \", text)\n",
        "    \n",
        "    # Optionally, shorten words to their stems\n",
        "    if stem_words:\n",
        "        text = text.split()\n",
        "        stemmer = SnowballStemmer('english')\n",
        "        stemmed_words = [stemmer.stem(word) for word in text]\n",
        "        text = \" \".join(stemmed_words)\n",
        "    \n",
        "    # Return a list of words\n",
        "    return(text)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6f3oFvlF4ruY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 108
        },
        "outputId": "1e9817bf-1476-4d1e-c8ad-7588afeb8945"
      },
      "source": [
        "import nltk\n",
        "nltk.download('wordnet')\n",
        "nltk.download('stopwords')"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/wordnet.zip.\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ATWctzrG4ruc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 198
        },
        "outputId": "dcb9929e-8939-40cc-dc20-f90eab500266"
      },
      "source": [
        "all_participants_mix = all_participants.copy()\n",
        "all_participants_mix['answer'] = all_participants_mix.apply(lambda row: text_to_wordlist(row.answer).split(), axis=1)\n",
        "all_participants_mix.head()"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>index</th>\n",
              "      <th>personId</th>\n",
              "      <th>question</th>\n",
              "      <th>answer</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>300.0</td>\n",
              "      <td>hi i'm ellie thanks for coming in today</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>300.0</td>\n",
              "      <td>i was created to talk to people in a safe and ...</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>300.0</td>\n",
              "      <td>think of me as a friend i don't judge i can't ...</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>300.0</td>\n",
              "      <td>i'm here to learn about people and would love ...</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>300.0</td>\n",
              "      <td>i'll ask a few questions to get us started and...</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   index  personId                                           question answer\n",
              "0      0     300.0            hi i'm ellie thanks for coming in today     []\n",
              "1      1     300.0  i was created to talk to people in a safe and ...     []\n",
              "2      2     300.0  think of me as a friend i don't judge i can't ...     []\n",
              "3      3     300.0  i'm here to learn about people and would love ...     []\n",
              "4      4     300.0  i'll ask a few questions to get us started and...     []"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pUafULKL4ruf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        },
        "outputId": "4e069a16-6ddb-4d6e-e771-821fdccc734d"
      },
      "source": [
        "print(all_participants_mix['answer'])"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0                                 []\n",
            "1                                 []\n",
            "2                                 []\n",
            "3                                 []\n",
            "4                                 []\n",
            "                    ...             \n",
            "14968                             []\n",
            "14969    [um, proud, fact, uh, give]\n",
            "14970                             []\n",
            "14971                             []\n",
            "14972                   [bye, thank]\n",
            "Name: answer, Length: 14973, dtype: object\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oFe1IwkF4rui",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "all_participants_mix_stopwords = all_participants.copy()\n",
        "all_participants_mix_stopwords['answer'] = all_participants_mix_stopwords.apply(lambda row: text_to_wordlist(row.answer, remove_stopwords=False).split(), axis=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BRizm4LX4run",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "e63be640-2e20-4c38-dae1-06acaee32e57"
      },
      "source": [
        "words = [w for w in all_participants_mix['answer'].tolist()]\n",
        "words = set(itertools.chain(*words))\n",
        "vocab_size = len(words)\n",
        "print(vocab_size)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "7373\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CbezqZ3j4ruq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "5e310ad2-79c4-473c-e325-6101d9199ee8"
      },
      "source": [
        "words_stop = [w for w in all_participants_mix_stopwords['answer'].tolist()]\n",
        "words_stop = set(itertools.chain(*words_stop))\n",
        "vocab_size_stop = len(words_stop)\n",
        "print(vocab_size_stop)"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "7449\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qu8EtI1F4rut",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 126
        },
        "outputId": "ce3df3e4-d984-45cd-cf6d-b01592bd260d"
      },
      "source": [
        "windows_size = WINDOWS_SIZE\n",
        "tokenizer = Tokenizer(num_words=vocab_size)\n",
        "tokenizer.fit_on_texts(all_participants_mix['answer'])\n",
        "tokenizer.fit_on_sequences(all_participants_mix['answer'])\n",
        "\n",
        "all_participants_mix['t_answer'] = tokenizer.texts_to_sequences(all_participants_mix['answer'])\n",
        "all_participants_mix['answer'].head()"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    []\n",
              "1    []\n",
              "2    []\n",
              "3    []\n",
              "4    []\n",
              "Name: answer, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F83kj8uQ4ruw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 198
        },
        "outputId": "c278defb-6af4-475f-a8c9-c5f15c562083"
      },
      "source": [
        "windows_size = WINDOWS_SIZE\n",
        "tokenizer = Tokenizer(num_words=vocab_size_stop)\n",
        "tokenizer.fit_on_texts(all_participants_mix_stopwords['answer'])\n",
        "tokenizer.fit_on_sequences(all_participants_mix_stopwords['answer'])\n",
        "\n",
        "all_participants_mix_stopwords['t_answer'] = tokenizer.texts_to_sequences(all_participants_mix_stopwords['answer'])\n",
        "all_participants_mix_stopwords.head()"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>index</th>\n",
              "      <th>personId</th>\n",
              "      <th>question</th>\n",
              "      <th>answer</th>\n",
              "      <th>t_answer</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>300.0</td>\n",
              "      <td>hi i'm ellie thanks for coming in today</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>300.0</td>\n",
              "      <td>i was created to talk to people in a safe and ...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>300.0</td>\n",
              "      <td>think of me as a friend i don't judge i can't ...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>300.0</td>\n",
              "      <td>i'm here to learn about people and would love ...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>300.0</td>\n",
              "      <td>i'll ask a few questions to get us started and...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   index  personId  ... answer t_answer\n",
              "0      0     300.0  ...     []       []\n",
              "1      1     300.0  ...     []       []\n",
              "2      2     300.0  ...     []       []\n",
              "3      3     300.0  ...     []       []\n",
              "4      4     300.0  ...     []       []\n",
              "\n",
              "[5 rows x 5 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dU8BCa7-4ru0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "d01ea771-9ded-4546-ae7a-3497a025a594"
      },
      "source": [
        "word_index = tokenizer.word_index\n",
        "word_size = len(word_index)\n",
        "print(word_index[\"sad\"])"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "652\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z6MMhiUz4ru3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 198
        },
        "outputId": "9041587a-230d-4a32-ff67-8c741a1c1629"
      },
      "source": [
        "all_participants_mix_stopwords.head()"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>index</th>\n",
              "      <th>personId</th>\n",
              "      <th>question</th>\n",
              "      <th>answer</th>\n",
              "      <th>t_answer</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>300.0</td>\n",
              "      <td>hi i'm ellie thanks for coming in today</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>300.0</td>\n",
              "      <td>i was created to talk to people in a safe and ...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>300.0</td>\n",
              "      <td>think of me as a friend i don't judge i can't ...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>300.0</td>\n",
              "      <td>i'm here to learn about people and would love ...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>300.0</td>\n",
              "      <td>i'll ask a few questions to get us started and...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   index  personId  ... answer t_answer\n",
              "0      0     300.0  ...     []       []\n",
              "1      1     300.0  ...     []       []\n",
              "2      2     300.0  ...     []       []\n",
              "3      3     300.0  ...     []       []\n",
              "4      4     300.0  ...     []       []\n",
              "\n",
              "[5 rows x 5 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "wasyqB3d4ru7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "windows_size = WINDOWS_SIZE\n",
        "cont = 0\n",
        "word_index = tokenizer\n",
        "phrases_lp_stop = pd.DataFrame(columns=['personId','answer', 't_answer'])\n",
        "answers = all_participants_mix_stopwords.groupby('personId').agg({'answer':'sum','t_answer':'sum'})\n",
        "\n",
        "#answers = all_participants_mix_stopwords.groupby('personId').agg({' ':\"sum\"},axis=\"columns\")\n",
        "\n",
        "#answers = all_participants_mix_stopwords.groupby('personId',axis=0).aggregate('sum')\n",
        "#answers1 = all_participants_mix_stopwords.groupby('personId',axis=0)\n",
        "\n",
        "#answers= answers1.DataFrame.aggregate('sum', axis=1)\n",
        "#print(answers[\"answer\"])\n",
        "print(phrases_lp_stop.head())\n",
        "#print(answers[1])\n",
        "#print(answers)\n",
        "#answers.head()\n",
        "#answers1.head()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RLdCHPyt4ru-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 254
        },
        "outputId": "e7caf346-d6f1-4e3c-99f5-445c30fd82b8"
      },
      "source": [
        "print(answers['answer'])"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "personId\n",
            "300.0    [good, atlanta, georgia, um, my, parent, are, ...\n",
            "301.0    [thank, you, mmm, k, i, am, doing, good, thank...\n",
            "302.0    [i, am, fine, how, about, yourself, i, am, fro...\n",
            "303.0    [okay, how, bout, yourself, here, in, californ...\n",
            "304.0    [i, am, doing, good, um, from, los, angeles, c...\n",
            "                               ...                        \n",
            "488.0    [yes, fine, oh, san, fernando, valley, uh, wel...\n",
            "489.0    [yes, i, am, doing, well, thank, you, san, lui...\n",
            "490.0    [yeah, i, am, doing, already, how, are, you, d...\n",
            "491.0    [yes, huh, overwhelmed, i, have, a, funeral, t...\n",
            "492.0    [yes, doing, pretty, good, thank, you, marylan...\n",
            "Name: answer, Length: 186, dtype: object\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hYLcpJk74rvA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "fc6c9b8d-d249-4d9a-e1c5-7b74c4b11a91"
      },
      "source": [
        "cont = 0\n",
        "for p in answers.iterrows():      \n",
        "    words = p[1][\"answer\"]\n",
        "    size = len(words)\n",
        "    word_tokens = p[1][\"t_answer\"]\n",
        "    print(cont)\n",
        "    for i in range(size):\n",
        "        sentence = words[i:min(i+windows_size,size)]  \n",
        "        tokens = word_tokens[i:min(i+windows_size,size)]  \n",
        "        phrases_lp_stop.loc[cont] = [p[0], sentence, tokens]\n",
        "        cont = cont + 1"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0\n",
            "366\n",
            "1910\n",
            "2537\n",
            "4564\n",
            "5586\n",
            "9031\n",
            "10645\n",
            "13317\n",
            "14253\n",
            "14978\n",
            "16167\n",
            "16840\n",
            "18094\n",
            "18864\n",
            "22270\n",
            "23681\n",
            "24333\n",
            "25004\n",
            "25734\n",
            "26438\n",
            "27229\n",
            "28181\n",
            "29783\n",
            "31179\n",
            "32089\n",
            "33961\n",
            "34490\n",
            "35308\n",
            "37387\n",
            "38393\n",
            "38893\n",
            "40421\n",
            "41348\n",
            "42739\n",
            "44279\n",
            "45715\n",
            "46737\n",
            "51500\n",
            "52373\n",
            "53416\n",
            "54025\n",
            "55336\n",
            "56086\n",
            "58111\n",
            "59615\n",
            "61954\n",
            "62412\n",
            "63425\n",
            "64967\n",
            "66450\n",
            "67651\n",
            "69384\n",
            "70677\n",
            "71189\n",
            "72243\n",
            "73920\n",
            "74343\n",
            "75023\n",
            "77376\n",
            "78019\n",
            "79352\n",
            "80184\n",
            "82410\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2645\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2646\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2647\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.Int64HashTable.get_item\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.Int64HashTable.get_item\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;31mKeyError\u001b[0m: 83492",
            "\nDuring handling of the above exception, another exception occurred:\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-29-1e4326a8fcb7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0msentence\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mwords\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mmin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mwindows_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0mtokens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mword_tokens\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mmin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mwindows_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m         \u001b[0mphrases_lp_stop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcont\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msentence\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtokens\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m         \u001b[0mcont\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcont\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m__setitem__\u001b[0;34m(self, key, value)\u001b[0m\n\u001b[1;32m    668\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    669\u001b[0m             \u001b[0mkey\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_if_callable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 670\u001b[0;31m         \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_setitem_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    671\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_setitem_with_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    672\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m_get_setitem_indexer\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m    655\u001b[0m         \u001b[0maxis\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maxis\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    656\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 657\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_convert_to_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    658\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    659\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m_convert_to_indexer\u001b[0;34m(self, obj, axis, raise_missing)\u001b[0m\n\u001b[1;32m   1691\u001b[0m         \u001b[0;31m# if we are a label return me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1692\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1693\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1694\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mLookupError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1695\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mABCMultiIndex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2644\u001b[0m                 )\n\u001b[1;32m   2645\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2646\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2647\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2648\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_cast_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZHq_gN-C4rvD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "phrases_lp_stop.head()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4aHSG2d74rvG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "windows_size = WINDOWS_SIZE\n",
        "cont = 0\n",
        "word_index = tokenizer\n",
        "phrases_lp = pd.DataFrame(columns=['personId','answer', 't_answer'])\n",
        "answers = all_participants_mix.groupby('personId').agg({'answer':'sum','t_answer':'sum'})\n",
        "\n",
        "for p in answers.iterrows():      \n",
        "    \n",
        "    words = p[1][\"answer\"]\n",
        "    size = len(words)\n",
        "    word_tokens = p[1][\"t_answer\"]\n",
        "    print(cont)\n",
        "    for i in range(size):\n",
        "        sentence = words[i:min(i+windows_size,size)]  \n",
        "        tokens = word_tokens[i:min(i+windows_size,size)]  \n",
        "        phrases_lp.loc[cont] = [p[0], sentence, tokens]\n",
        "        cont = cont + 1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cn_n6Bn44rvI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "phrases_lp.to_csv('phrases_lp.csv', sep='\\t')\n",
        "print(\"File was created\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sLdXWv674rvL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "phrases_lp.head()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D3clw5bO4rvO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "phrases_lp[\"t_answer\"] = pad_sequences(phrases_lp[\"t_answer\"], value=0, padding=\"post\", maxlen=windows_size).tolist()\n",
        "phrases_lp.head()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4-Tn_3os4rvR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "phrases_lp_stop.to_csv( 'phrases_lp_stop.csv', sep='\\t')\n",
        "print(\"File was created\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zZYbJ_r-4rvV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "phrases_lp = pd.read_csv( 'phrases_lp.csv', sep='\\t', converters={\"t_answer\": literal_eval}) \n",
        "phrases_lp.head()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "que1vBel4rvZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def load_avec_dataset_file(path, score_column):\n",
        "    ds = pd.read_csv(path, sep=',')\n",
        "    ds['level'] = pd.cut(ds[score_column], bins=[-1,0,5,10,15,25], labels=[0,1,2,3,4])\n",
        "    ds['PHQ8_Score'] = ds[score_column]\n",
        "    ds['cat_level'] = keras.utils.to_categorical(ds['level'], num_classes).tolist()\n",
        "    ds = ds[['Participant_ID', 'level', 'cat_level', 'PHQ8_Score']]\n",
        "    ds = ds.astype({\"Participant_ID\": float, \"level\": int, 'PHQ8_Score': int})\n",
        "    return ds"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QRl6REaA4rvb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 216
        },
        "outputId": "b4bb15bb-dfc9-4116-a31a-ccce85d31196"
      },
      "source": [
        "train = load_avec_dataset_file('train_split_Depression_AVEC2017.csv', 'PHQ8_Score')\n",
        "dev = load_avec_dataset_file('dev_split_Depression_AVEC2017.csv', 'PHQ8_Score')\n",
        "test = load_avec_dataset_file('full_test_split.csv', 'PHQ_Score')\n",
        "print(\"Size: train= {}, dev= {}, test= {}\".format(len(train), len(dev), len(test)))\n",
        "train.head()"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Size: train= 107, dev= 35, test= 47\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Participant_ID</th>\n",
              "      <th>level</th>\n",
              "      <th>cat_level</th>\n",
              "      <th>PHQ8_Score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>303.0</td>\n",
              "      <td>0</td>\n",
              "      <td>[1.0, 0.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>304.0</td>\n",
              "      <td>2</td>\n",
              "      <td>[0.0, 0.0, 1.0, 0.0, 0.0]</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>305.0</td>\n",
              "      <td>2</td>\n",
              "      <td>[0.0, 0.0, 1.0, 0.0, 0.0]</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>310.0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0.0, 1.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>312.0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0.0, 1.0, 0.0, 0.0, 0.0]</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Participant_ID  level                  cat_level  PHQ8_Score\n",
              "0           303.0      0  [1.0, 0.0, 0.0, 0.0, 0.0]           0\n",
              "1           304.0      2  [0.0, 0.0, 1.0, 0.0, 0.0]           6\n",
              "2           305.0      2  [0.0, 0.0, 1.0, 0.0, 0.0]           7\n",
              "3           310.0      1  [0.0, 1.0, 0.0, 0.0, 0.0]           4\n",
              "4           312.0      1  [0.0, 1.0, 0.0, 0.0, 0.0]           2"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yxIBdAfD4rve",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "7907913d-355e-46ad-fdce-807697388dfe"
      },
      "source": [
        "ds_total = pd.concat([train,dev,test])\n",
        "total_phq8 = len(ds_total)\n",
        "print(\"Total size = {}\".format(total_phq8))"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Total size = 189\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wg2xkxPA4rvg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "19e94ce4-e17a-4b7b-e88a-f14d119590a3"
      },
      "source": [
        "ds_total.to_csv( 'ds_total.csv', sep = '\\t')\n",
        "print('done')"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "done\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DATKVll24rvk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def split_by_phq_level(ds):\n",
        "    none_ds = ds[ds['level']==0]\n",
        "    mild_ds = ds[ds['level']==1]\n",
        "    moderate_ds = ds[ds['level']==2]\n",
        "    moderate_severe_ds = ds[ds['level']==3]\n",
        "    severe_ds = ds[ds['level']==4]\n",
        "    return (none_ds, mild_ds, moderate_ds, moderate_severe_ds, severe_ds)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Bli9lZ14rvo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "e8b16fd3-ed8b-46dd-f96b-6c375661c3c2"
      },
      "source": [
        "none_ds, mild_ds, moderate_ds, moderate_severe_ds, severe_ds = split_by_phq_level(ds_total)\n",
        "print(\"Quantity per none_ds: {}, mild_ds: {}, moderate_ds {}, moderate_severe_ds: {}, severe_ds {}\".format(len(none_ds), len(mild_ds), len(moderate_ds), len(moderate_severe_ds), len(severe_ds)))"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Quantity per none_ds: 26, mild_ds: 70, moderate_ds 47, moderate_severe_ds: 24, severe_ds 22\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O5vTK_mZ4rvw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "e6a10604-136f-463a-8901-a0a3a65bc68c"
      },
      "source": [
        "b_none_ds = ds_total[ds_total['level']==0]\n",
        "b_mild_ds = ds_total[ds_total['level']==1].sample(26)\n",
        "b_moderate_ds = ds_total[ds_total['level']==2].sample(26)\n",
        "b_moderate_severe_ds = ds_total[ds_total['level']==3]\n",
        "b_severe_ds = ds_total[ds_total['level']==4]\n",
        "\n",
        "ds_total_b = pd.concat([b_none_ds, b_mild_ds, b_moderate_ds, b_moderate_severe_ds, b_severe_ds])\n",
        "print(\"Quantity per none_ds: {}, mild_ds: {}, moderate_ds {}, moderate_severe_ds: {}, severe_ds {}\".format(len(b_none_ds), len(b_mild_ds), len(b_moderate_ds), len(b_moderate_severe_ds), len(b_severe_ds)))"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Quantity per none_ds: 26, mild_ds: 26, moderate_ds 26, moderate_severe_ds: 24, severe_ds 22\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vk6-KOKd4rv0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 189
        },
        "outputId": "f81d4883-55cf-4cdc-a021-9c3a8b9bfc52"
      },
      "source": [
        "ds_lp = pd.merge(ds_total, phrases_lp,left_on='Participant_ID', right_on='personId')\n",
        "ds_lp_b = pd.merge(ds_total_b, phrases_lp,left_on='Participant_ID', right_on='personId')"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-42-9492d88ad5af>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mds_lp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmerge\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mds_total\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mphrases_lp\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mleft_on\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Participant_ID'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mright_on\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'personId'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mds_lp_b\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmerge\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mds_total_b\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mphrases_lp\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mleft_on\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Participant_ID'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mright_on\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'personId'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'phrases_lp' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YZOXXQHm4rv5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def distribute_instances(ds, split_in = [70,14,16]):\n",
        "    ds_shuffled = ds.sample(frac=1)\n",
        "    none_ds, mild_ds, moderate_ds, moderate_severe_ds, severe_ds = split_by_phq_level(ds_shuffled)\n",
        "    eq_ds = dict()\n",
        "    prev_none = prev_mild = prev_moderate = prev_moderate_severe = prev_severe = 0\n",
        "    split = split_in\n",
        "    for p in split:\n",
        "        last_none = min(len(none_ds), prev_none + round(len(none_ds) * p/100))\n",
        "        last_mild = min(len(mild_ds), prev_mild + round(len(mild_ds) * p/100))\n",
        "        last_moderate = min(len(moderate_ds), prev_moderate + round(len(moderate_ds) * p/100))\n",
        "        last_moderate_severe = min(len(moderate_severe_ds), prev_moderate_severe + round(len(moderate_severe_ds) * p/100))\n",
        "        last_severe = min(len(severe_ds), prev_severe + round(len(severe_ds) * p/100))  \n",
        "        eq_ds['d'+str(p)] = pd.concat([none_ds[prev_none: last_none], mild_ds[prev_mild: last_mild], moderate_ds[prev_moderate: last_moderate], moderate_severe_ds[prev_moderate_severe: last_moderate_severe], severe_ds[prev_severe: last_severe]])\n",
        "        prev_none = last_none\n",
        "        prev_mild = last_mild\n",
        "        prev_moderate = last_moderate\n",
        "        prev_moderate_severe = last_moderate_severe\n",
        "        prev_severe = last_severe  \n",
        "    return (eq_ds['d70'], eq_ds['d14'], eq_ds['d16'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mj1hGvg24rv_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 189
        },
        "outputId": "f0a605e5-63bb-4099-8a6e-f1511ecdf071"
      },
      "source": [
        "train_lp, dev_lp, test_lp = distribute_instances(ds_lp)\n",
        "train_lp_b, dev_lp_b, test_lp_b = distribute_instances(ds_lp_b)"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-45-b034f32dce4b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrain_lp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdev_lp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_lp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdistribute_instances\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mds_lp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mtrain_lp_b\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdev_lp_b\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_lp_b\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdistribute_instances\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mds_lp_b\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'ds_lp' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CIaOH3OE4rwD",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 171
        },
        "outputId": "0e3d5aaf-cd5a-4454-b141-e1d2fd141663"
      },
      "source": [
        "train_lp.tail()"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-49-7501587a2b3f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrain_lp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtail\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'train_lp' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NyuxNdkU4rwG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def confusion_matrix(model, x, y):\n",
        "    prediction = model.predict(x, batch_size=None, verbose=0, steps=None)\n",
        "    labels=['none','mild','moderate','moderately severe', 'severe']\n",
        "\n",
        "    max_prediction = np.argmax(prediction, axis=1)\n",
        "    max_actual = np.argmax(y, axis=1)\n",
        "\n",
        "    y_pred = pd.Categorical.from_codes(max_prediction, labels)\n",
        "    y_actu = pd.Categorical.from_codes(max_actual, labels)\n",
        "\n",
        "    return pd.crosstab(y_actu, y_pred)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JTdT97DV4rwI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 244
        },
        "outputId": "153fe41f-9468-4ad1-ef0f-45071001e86e"
      },
      "source": [
        "embeddings_index = dict()\n",
        "f = open('/content/drive/My Drive/glove/glove.6B.100d.txt', encoding=\"utf8\")\n",
        "for line in f:\n",
        "    values = line.split()\n",
        "    word = values[0]\n",
        "    coefs = np.asarray(values[1:], dtype='float32')\n",
        "    embeddings_index[word] = coefs\n",
        "f.close()"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "error",
          "ename": "FileNotFoundError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-50-c234793831b0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0membeddings_index\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/content/drive/My Drive/glove/glove.6B.100d.txt'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoding\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"utf8\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mline\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mline\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mword\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvalues\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/content/drive/My Drive/glove/glove.6B.100d.txt'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UBf1Zwjl1Ctm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 128
        },
        "outputId": "75fb6d65-24ba-4340-a281-ad99853d4a4d"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J-C8xuof4rwM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def fill_embedding_matrix(tokenizer):\n",
        "    vocab_size = len(tokenizer.word_index)\n",
        "    embedding_matrix = np.zeros((vocab_size+1, 100))\n",
        "    for word, i in tokenizer.word_index.items():\n",
        "        embedding_vector = embeddings_index.get(word)\n",
        "        if embedding_vector is not None:        \n",
        "            embedding_matrix[i] = embedding_vector\n",
        "    return embedding_matrix"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C52oAoHQ4rwP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "embedding_matrix_lp = fill_embedding_matrix(tokenizer)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yS6O85Rx4rwU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_lp.head()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MbL1V1Dz4rwZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_lp_copy = train_lp\n",
        "print(len(train_lp_copy))\n",
        "for index,row in train_lp_copy.iterrows():\n",
        "    if len(row['t_answer'])<10:\n",
        "        train_lp_copy.drop(index,inplace=True)\n",
        "print(len(train_lp_copy))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g6a03bs14rwd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_lp = train_lp_copy"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XKzhEhiP4rwf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "dev_lp_copy = dev_lp\n",
        "print(len(dev_lp_copy))\n",
        "for index,row in dev_lp_copy.iterrows():\n",
        "    if len(row['t_answer'])<10:\n",
        "        dev_lp_copy.drop(index,inplace=True)\n",
        "print(len(dev_lp_copy))  \n",
        "dev_lp = dev_lp_copy"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-p-oOCtZ4rwh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test_lp_copy = test_lp\n",
        "print(len(test_lp_copy))\n",
        "for index,row in test_lp_copy.iterrows():\n",
        "    if len(row['t_answer'])<10:\n",
        "        test_lp_copy.drop(index,inplace=True)\n",
        "print(len(test_lp_copy))  \n",
        "test_lp = test_lp_copy"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "009XR2QW4rwk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_a = np.stack(train_lp['t_answer'], axis=0)\n",
        "dev_a = np.stack(dev_lp['t_answer'], axis=0)\n",
        "train_y = np.stack(train_lp['cat_level'], axis=0)\n",
        "dev_y = np.stack(dev_lp['cat_level'], axis=0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kSS2yeEJ4rwm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_lp_b_copy = train_lp_b\n",
        "print(len(train_lp_b_copy))\n",
        "for index,row in train_lp_b_copy.iterrows():\n",
        "    if len(row['t_answer'])<10:\n",
        "        train_lp_b_copy.drop(index,inplace=True)\n",
        "print(len(train_lp_b_copy))  \n",
        "train_lp_b = train_lp_b_copy"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zxst4lUr4rwr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "dev_lp_b_copy = dev_lp_b\n",
        "print(len(dev_lp_b_copy))\n",
        "for index,row in dev_lp_b_copy.iterrows():\n",
        "    if len(row['t_answer'])<10:\n",
        "        dev_lp_b_copy.drop(index,inplace=True)\n",
        "print(len(dev_lp_b_copy))  \n",
        "dev_lp_b = dev_lp_b_copy"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bJcfXM6V4rwx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test_lp_b_copy = test_lp_b\n",
        "print(len(test_lp_b_copy))\n",
        "for index,row in test_lp_b_copy.iterrows():\n",
        "    if len(row['t_answer'])<10:\n",
        "        test_lp_b_copy.drop(index,inplace=True)\n",
        "print(len(test_lp_b_copy))  \n",
        "test_lp_b = test_lp_b_copy"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8ThLUtdx4rw1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_a_b = np.stack(train_lp_b['t_answer'], axis=0)\n",
        "dev_a_b = np.stack(dev_lp_b['t_answer'], axis=0)\n",
        "train_y_b = np.stack(train_lp_b['cat_level'], axis=0)\n",
        "dev_y_b = np.stack(dev_lp_b['cat_level'], axis=0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "EQSCAtItLccU",
        "colab": {}
      },
      "source": [
        "early_stopping = EarlyStopping(monitor='val_loss', patience=3)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "0DAAvG-TKgLS",
        "colab": {}
      },
      "source": [
        "answer_inp = Input(shape=(windows_size, ))\n",
        "embedding_size_glove = 100\n",
        "answer_emb1 = Embedding(vocab_size_stop+1, embedding_size_glove, weights=[embedding_matrix_lp], input_length=windows_size, trainable=False)(answer_inp)\n",
        "\n",
        "bt = BatchNormalization()(answer_emb1)\n",
        "lstm = LSTM(embedding_size_glove, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)(bt)\n",
        "\n",
        "dense1 = Dense(units=256, activation=\"relu\")(lstm)\n",
        "dense2 = Dense(units=256, activation=\"relu\")(dense1)\n",
        "\n",
        "flatten = Flatten()(dense2)\n",
        "\n",
        "out = Dense(5,  activation='softmax')(flatten)\n",
        "\n",
        "model = Model(inputs=[answer_inp], outputs=[out])\n",
        "model.compile(optimizer=\"adam\", loss='categorical_crossentropy', metrics=['accuracy'])\n",
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "frVCIJUx4rw_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_glove_lstm_hist = model.fit(train_a, train_y, validation_data=(dev_a, dev_y), epochs=30, batch_size=64, shuffle=True, callbacks=[early_stopping])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RYdTu2y04rxC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_glove_lstm_hist_b = model.fit(train_a_b, train_y_b, validation_data=(dev_a_b, dev_y_b), epochs=30, batch_size=64, shuffle=True, callbacks=[early_stopping])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BiQ7DnjX4rxE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test_a = np.stack(test_lp['t_answer'], axis=0)\n",
        "test_y = np.stack(test_lp['cat_level'], axis=0)\n",
        "test_a_b = np.stack(test_lp_b['t_answer'], axis=0)\n",
        "test_y_b = np.stack(test_lp_b['cat_level'], axis=0)\n",
        "df_confusion = confusion_matrix(model, test_a_b, test_y_b)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qjyFmZBw4rxH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_confusion"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FjBIvwQ24rxL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.save('/media/prahlad/New Volume/DIAC-WOZ/model_glove_lstm_b.h5')\n",
        "json_dict = model_glove_lstm_hist.history\n",
        "with open('/media/prahlad/New Volume/DIAC-WOZ/model_glove_lstm_b_hist.json', 'w') as f:\n",
        "    f.write(str(json_dict))\n",
        "print('File got created')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WesKwUcR4rxO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from keras.models import load_model\n",
        "model = load_model('/media/prahlad/New Volume/DIAC-WOZ/model_glove_lstm_b.h5')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f0yh4Y7r4rxR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "plot_loss(model_glove_lstm_hist)\n",
        "plot_acc(model_glove_lstm_hist)\n",
        "plot_loss(model_glove_lstm_hist_b)\n",
        "plot_acc(model_glove_lstm_hist_b)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qW8OVU134rxS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def test_model(text, model):\n",
        "    print(text)\n",
        "    word_list = text_to_wordlist(text)\n",
        "    sequences = tokenizer.texts_to_sequences([word_list])\n",
        "    sequences_input = list(itertools.chain(*sequences))\n",
        "    sequences_input =  pad_sequences([sequences_input], value=0, padding=\"post\", maxlen=windows_size).tolist()\n",
        "    print(sequences_input)\n",
        "    input_a = np.asarray(sequences_input)\n",
        "    pred = model.predict(input_a, batch_size=None, verbose=0, steps=None)\n",
        "    predicted_class = np.argmax(pred)\n",
        "    print(labels[predicted_class])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MfRbc_Bi4rxU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sen = \"All is going right with the party, I'm happy to know new people\"\n",
        "test_model(sen, model)\n",
        "sen = \"I want an ice cream and have some fries for lunch\"\n",
        "test_model(sen, model)\n",
        "sen = \"I'm afraid of losing my work, I don't have any money\"\n",
        "test_model(sen, model)\n",
        "sen = \"I'm worried about my future, I'm afraid of it\"\n",
        "test_model(sen, model)\n",
        "sen = \"I am a graduate student\"\n",
        "test_model(sen, model)\n",
        "sen = \"I am getting married\"\n",
        "test_model(sen, model)\n",
        "sen = \"This party is great, I know lots of people\"\n",
        "test_model(sen, model)\n",
        "sen = \"I miss my parents, brothers and sisters\"\n",
        "test_model(sen, model)\n",
        "sen = \"I detest my horrible job\"\n",
        "test_model(sen, model)\n",
        "sen = \"suicide\"\n",
        "test_model(sen, model)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2Qk_nFVe4rxW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "answer_inp = Input(shape=(windows_size, ))\n",
        "embedding_size_glove = 100\n",
        "answer_emb1 = Embedding(vocab_size_stop+1, embedding_size_glove, weights=[embedding_matrix_lp], input_length=windows_size, trainable=False)(answer_inp)\n",
        "\n",
        "\n",
        "lstm1 = LSTM(embedding_size_glove, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)(answer_emb1)\n",
        "lstm2 = LSTM(embedding_size_glove, dropout=0.2, recurrent_dropout=0.2)(lstm1)\n",
        "\n",
        "X = Dropout(0.2)(lstm2)\n",
        "bt = BatchNormalization()(X)\n",
        "dense1 = Dense(units=256, activation=\"relu\")(bt)\n",
        "\n",
        "out = Dense(5,  activation='softmax')(dense1)\n",
        "\n",
        "model_2lstm = Model(inputs=[answer_inp], outputs=[out])\n",
        "model_2lstm.compile(optimizer=\"adam\", loss='categorical_crossentropy', metrics=['accuracy'])\n",
        "model_2lstm.summary()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KAU2mESm4rxZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_glove_2lstm_b_hist = model_2lstm.fit(train_a_b, train_y_b, validation_data=(dev_a_b, dev_y_b), epochs=30, batch_size=64, shuffle=True, callbacks=[early_stopping])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U_S4M61b4rxb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "plot_loss(model_glove_2lstm_b_hist)\n",
        "plot_acc(model_glove_2lstm_b_hist)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DDcreSBX4rxd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "score = model_2lstm.evaluate(test_a_b, test_y_b, verbose=0)\n",
        "print('Test loss:', score[0])\n",
        "print('Test accuracy:', score[1])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZYINbsZD4rxh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_confusion = confusion_matrix(model_2lstm, test_a_b, test_y_b)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GO5dLUjs4rxj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_confusion\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1T6tD8Ma4rxl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.save('/media/prahlad/New Volume/DIAC-WOZ/model_glove_2lstm_b.h5')\n",
        "json_dict = model_glove_2lstm_b_hist.history\n",
        "with open('/media/prahlad/New Volume/DIAC-WOZ/model_glove_2lstm_b_hist.json', 'w') as f:\n",
        "    f.write(str(json_dict))\n",
        "print('File got created')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3QfGn5qo4rxn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sen = \"All is going right with the party, I'm happy to know new people\"\n",
        "test_model(sen, model_2lstm)\n",
        "sen = \"I want an ice cream and have some fries for lunch\"\n",
        "test_model(sen, model_2lstm)\n",
        "sen = \"I'm afraid of losing my work, I don't have any money\"\n",
        "test_model(sen, model_2lstm)\n",
        "sen = \"I'm worried about my future, I'm afraid of it\"\n",
        "test_model(sen, model_2lstm)\n",
        "sen = \"My father loves me\"\n",
        "test_model(sen, model_2lstm)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u_m_GMz24rxo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sen = \"Please kill me\"\n",
        "test_model(sen, model_2lstm)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}