TrainModel.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Python 3+\n",
    "\n",
    "# 3rd party imports (not present in the standard python library)\n",
    "# To install, pip install numpy pandas\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# Standard python library imports\n",
    "\n",
    "import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['training.1600000.processed.noemoticon.csv']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A large dataset with 1.6 million tweets are being used to train the model\n",
    "# Due to its size, the file is not included in this repository\n",
    "# The dataset can be downloaded from https://www.kaggle.com/kazanova/sentiment140\n",
    "\n",
    "# File in current workspace\n",
    "glob.glob('*.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import data\n",
    "\n",
    "df = pd.read_csv('training.1600000.processed.noemoticon.csv', encoding = 'ISO-8859-1', names = [\"Score\", \"Id\", \"Date\", \"Flag\", \"User\", \"Tweet\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Score</th>\n",
       "      <th>Id</th>\n",
       "      <th>Date</th>\n",
       "      <th>Flag</th>\n",
       "      <th>User</th>\n",
       "      <th>Tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1467810369</td>\n",
       "      <td>Mon Apr 06 22:19:45 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>_TheSpecialOne_</td>\n",
       "      <td>@switchfoot http://twitpic.com/2y1zl - Awww, t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1467810672</td>\n",
       "      <td>Mon Apr 06 22:19:49 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>scotthamilton</td>\n",
       "      <td>is upset that he can't update his Facebook by ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1467810917</td>\n",
       "      <td>Mon Apr 06 22:19:53 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>mattycus</td>\n",
       "      <td>@Kenichan I dived many times for the ball. Man...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1467811184</td>\n",
       "      <td>Mon Apr 06 22:19:57 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>ElleCTF</td>\n",
       "      <td>my whole body feels itchy and like its on fire</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1467811193</td>\n",
       "      <td>Mon Apr 06 22:19:57 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>Karoli</td>\n",
       "      <td>@nationwideclass no, it's not behaving at all....</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Score          Id                          Date      Flag             User  \\\n",
       "0      0  1467810369  Mon Apr 06 22:19:45 PDT 2009  NO_QUERY  _TheSpecialOne_   \n",
       "1      0  1467810672  Mon Apr 06 22:19:49 PDT 2009  NO_QUERY    scotthamilton   \n",
       "2      0  1467810917  Mon Apr 06 22:19:53 PDT 2009  NO_QUERY         mattycus   \n",
       "3      0  1467811184  Mon Apr 06 22:19:57 PDT 2009  NO_QUERY          ElleCTF   \n",
       "4      0  1467811193  Mon Apr 06 22:19:57 PDT 2009  NO_QUERY           Karoli   \n",
       "\n",
       "                                               Tweet  \n",
       "0  @switchfoot http://twitpic.com/2y1zl - Awww, t...  \n",
       "1  is upset that he can't update his Facebook by ...  \n",
       "2  @Kenichan I dived many times for the ball. Man...  \n",
       "3    my whole body feels itchy and like its on fire   \n",
       "4  @nationwideclass no, it's not behaving at all....  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# First 5 records\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Score</th>\n",
       "      <th>Id</th>\n",
       "      <th>Date</th>\n",
       "      <th>Flag</th>\n",
       "      <th>User</th>\n",
       "      <th>Tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1599995</th>\n",
       "      <td>4</td>\n",
       "      <td>2193601966</td>\n",
       "      <td>Tue Jun 16 08:40:49 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>AmandaMarie1028</td>\n",
       "      <td>Just woke up. Having no school is the best fee...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1599996</th>\n",
       "      <td>4</td>\n",
       "      <td>2193601969</td>\n",
       "      <td>Tue Jun 16 08:40:49 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>TheWDBoards</td>\n",
       "      <td>TheWDB.com - Very cool to hear old Walt interv...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1599997</th>\n",
       "      <td>4</td>\n",
       "      <td>2193601991</td>\n",
       "      <td>Tue Jun 16 08:40:49 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>bpbabe</td>\n",
       "      <td>Are you ready for your MoJo Makeover? Ask me f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1599998</th>\n",
       "      <td>4</td>\n",
       "      <td>2193602064</td>\n",
       "      <td>Tue Jun 16 08:40:49 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>tinydiamondz</td>\n",
       "      <td>Happy 38th Birthday to my boo of alll time!!! ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1599999</th>\n",
       "      <td>4</td>\n",
       "      <td>2193602129</td>\n",
       "      <td>Tue Jun 16 08:40:50 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>RyanTrevMorris</td>\n",
       "      <td>happy #charitytuesday @theNSPCC @SparksCharity...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Score          Id                          Date      Flag  \\\n",
       "1599995      4  2193601966  Tue Jun 16 08:40:49 PDT 2009  NO_QUERY   \n",
       "1599996      4  2193601969  Tue Jun 16 08:40:49 PDT 2009  NO_QUERY   \n",
       "1599997      4  2193601991  Tue Jun 16 08:40:49 PDT 2009  NO_QUERY   \n",
       "1599998      4  2193602064  Tue Jun 16 08:40:49 PDT 2009  NO_QUERY   \n",
       "1599999      4  2193602129  Tue Jun 16 08:40:50 PDT 2009  NO_QUERY   \n",
       "\n",
       "                    User                                              Tweet  \n",
       "1599995  AmandaMarie1028  Just woke up. Having no school is the best fee...  \n",
       "1599996      TheWDBoards  TheWDB.com - Very cool to hear old Walt interv...  \n",
       "1599997           bpbabe  Are you ready for your MoJo Makeover? Ask me f...  \n",
       "1599998     tinydiamondz  Happy 38th Birthday to my boo of alll time!!! ...  \n",
       "1599999   RyanTrevMorris  happy #charitytuesday @theNSPCC @SparksCharity...  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Last 5 records\n",
    "\n",
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To train the model, our primary data points are the tweet and the score assoociated with the score\n",
    "# Score here is the sentiment where 0 = negative, 4 = positive\n",
    "# Columns that are not required are removed and the score is normalized to be in the 0 - 1 range\n",
    "\n",
    "df.drop([\"Id\", \"Date\", \"Flag\", \"User\"], axis = 1, inplace = True)\n",
    "df['Score'] = df['Score'].apply(lambda i : i / 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Score</th>\n",
       "      <th>Tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>@switchfoot http://twitpic.com/2y1zl - Awww, t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>is upset that he can't update his Facebook by ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>@Kenichan I dived many times for the ball. Man...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>my whole body feels itchy and like its on fire</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>@nationwideclass no, it's not behaving at all....</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Score                                              Tweet\n",
       "0    0.0  @switchfoot http://twitpic.com/2y1zl - Awww, t...\n",
       "1    0.0  is upset that he can't update his Facebook by ...\n",
       "2    0.0  @Kenichan I dived many times for the ball. Man...\n",
       "3    0.0    my whole body feels itchy and like its on fire \n",
       "4    0.0  @nationwideclass no, it's not behaving at all...."
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"@switchfoot http://twitpic.com/2y1zl - Awww, that's a bummer.  You shoulda got David Carr of Third Day to do it. ;D\""
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Tweet'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Tweet cleanup (this process takes a significant amount of time)\n",
    "# Use the df-cleaned.pickle to load a cleaned up dataframe\n",
    "# Removing stop words, @ mentions, webpages and special characters\n",
    "\n",
    "from nltk.corpus import stopwords # nltk.download('stopwords') before importing\n",
    "from nltk.stem import PorterStemmer\n",
    "stemmer = PorterStemmer()\n",
    "\n",
    "def clean(tweet):\n",
    "    stage1 = [word for word in tweet.lower().split() if word not in stopwords.words('english')] # stopword removal\n",
    "    stage2 = [word[1:] if word.startswith('#') else word for word in stage1] # Hashtag symbol removal\n",
    "    stage3 = [stemmer.stem(word) for word in stage2 if not any([word.startswith('@'), word.startswith('http'), word.startswith('www')])] # @ mentions and websites removal and stemming\n",
    "    return ' '.join(stage3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 33min 5s, sys: 7min 23s, total: 40min 28s\n",
      "Wall time: 40min 32s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "df['TweetStripped'] = df['Tweet'].apply(clean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Score</th>\n",
       "      <th>Tweet</th>\n",
       "      <th>TweetStripped</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>@switchfoot http://twitpic.com/2y1zl - Awww, t...</td>\n",
       "      <td>- awww, that' bummer. shoulda got david carr t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>is upset that he can't update his Facebook by ...</td>\n",
       "      <td>upset can't updat facebook text it... might cr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>@Kenichan I dived many times for the ball. Man...</td>\n",
       "      <td>dive mani time ball. manag save 50% rest go bound</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>my whole body feels itchy and like its on fire</td>\n",
       "      <td>whole bodi feel itchi like fire</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>@nationwideclass no, it's not behaving at all....</td>\n",
       "      <td>no, behav all. i'm mad. here? can't see there.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Score                                              Tweet  \\\n",
       "0    0.0  @switchfoot http://twitpic.com/2y1zl - Awww, t...   \n",
       "1    0.0  is upset that he can't update his Facebook by ...   \n",
       "2    0.0  @Kenichan I dived many times for the ball. Man...   \n",
       "3    0.0    my whole body feels itchy and like its on fire    \n",
       "4    0.0  @nationwideclass no, it's not behaving at all....   \n",
       "\n",
       "                                       TweetStripped  \n",
       "0  - awww, that' bummer. shoulda got david carr t...  \n",
       "1  upset can't updat facebook text it... might cr...  \n",
       "2  dive mani time ball. manag save 50% rest go bound  \n",
       "3                    whole bodi feel itchi like fire  \n",
       "4     no, behav all. i'm mad. here? can't see there.  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import dataframe from pickle\n",
    "\n",
    "import pickle\n",
    "\n",
    "with open('Pickled data/df-cleaned-final.pickle', 'rb') as f:\n",
    "    df = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Score</th>\n",
       "      <th>Tweet</th>\n",
       "      <th>TweetStripped</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>@switchfoot http://twitpic.com/2y1zl - Awww, t...</td>\n",
       "      <td>- awww, that' bummer. shoulda got david carr t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>is upset that he can't update his Facebook by ...</td>\n",
       "      <td>upset can't updat facebook text it... might cr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>@Kenichan I dived many times for the ball. Man...</td>\n",
       "      <td>dive mani time ball. manag save 50% rest go bound</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>my whole body feels itchy and like its on fire</td>\n",
       "      <td>whole bodi feel itchi like fire</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>@nationwideclass no, it's not behaving at all....</td>\n",
       "      <td>no, behav all. i'm mad. here? can't see there.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Score                                              Tweet  \\\n",
       "0    0.0  @switchfoot http://twitpic.com/2y1zl - Awww, t...   \n",
       "1    0.0  is upset that he can't update his Facebook by ...   \n",
       "2    0.0  @Kenichan I dived many times for the ball. Man...   \n",
       "3    0.0    my whole body feels itchy and like its on fire    \n",
       "4    0.0  @nationwideclass no, it's not behaving at all....   \n",
       "\n",
       "                                       TweetStripped  \n",
       "0  - awww, that' bummer. shoulda got david carr t...  \n",
       "1  upset can't updat facebook text it... might cr...  \n",
       "2  dive mani time ball. manag save 50% rest go bound  \n",
       "3                    whole bodi feel itchi like fire  \n",
       "4     no, behav all. i'm mad. here? can't see there.  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Analysis\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer # Perfoms the TF-IDF\n",
    "from sklearn.model_selection import train_test_split # Used to split the data into training and testing\n",
    "\n",
    "# Data is split in the ratio of 0.9 (train) : 0.1 (test)\n",
    "train_x, test_x, train_y, test_y = train_test_split(df['TweetStripped'], df['Score'], test_size = 0.1, shuffle = True)\n",
    "\n",
    "# To compare the accuracy when the raw tweet is used to train the model, the original data is split as well\n",
    "train_x2, test_x2, train_y2, test_y2 = train_test_split(df['Tweet'], df['Score'], test_size = 0.1, shuffle = True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "CPU times: user 36.5 s, sys: 816 ms, total: 37.3 s\nWall time: 36.6 s\n"
    },
    {
     "data": {
      "text/plain": "TfidfVectorizer(analyzer='word', binary=False, decode_error='strict',\n                dtype=<class 'numpy.float64'>, encoding='utf-8',\n                input='content', lowercase=True, max_df=1.0, max_features=10000,\n                min_df=1, ngram_range=(1, 2), norm='l2', preprocessor=None,\n                smooth_idf=True, stop_words='english', strip_accents=None,\n                sublinear_tf=False, token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n                tokenizer=None, use_idf=True, vocabulary=None)"
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Initialize and fit the TfTfidfVectorizer\n",
    "vector = TfidfVectorizer(max_features = 10000, ngram_range = (1,2), stop_words='english')\n",
    "%time vector.fit(train_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transform the data to pass it into various classifiers\n",
    "train_x_transformed = vector.transform(train_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The data will be trained on several models to find the one with the highest accuracy\n",
    "\n",
    "from sklearn import model_selection\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "models = []\n",
    "models.append(('LR', LogisticRegression()))\n",
    "models.append(('NB', MultinomialNB()))\n",
    "\n",
    "### Models below for this dataset take significantly longer\n",
    "#models.append(('LDA', LinearDiscriminantAnalysis()))\n",
    "#models.append(('KNN', KNeighborsClassifier()))\n",
    "#models.append(('CART', DecisionTreeClassifier()))\n",
    "#models.append(('SVM', SVC()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  10 out of  10 | elapsed:   39.8s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR: Average: 0.7687840277777778, std: 0.0010597752371332022\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NB: Average: 0.7535305555555556, std: 0.001792321155222122\n",
      "CPU times: user 1.96 s, sys: 1.02 s, total: 2.98 s\n",
      "Wall time: 43.5 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  10 out of  10 | elapsed:    3.2s finished\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# Train the models\n",
    "\n",
    "results = dict()\n",
    "scoring = 'accuracy'\n",
    "\n",
    "for name, model in models:\n",
    "    kfold = model_selection.KFold(n_splits = 10, random_state = 9)\n",
    "    cv_results = model_selection.cross_val_score(model, train_x_transformed, train_y, cv = kfold, scoring = 'accuracy', n_jobs = -1, verbose = 1)\n",
    "    results[name] = cv_results\n",
    "    print('{}: Average: {}, std: {}'.format(name, cv_results.mean(), cv_results.std()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neural_network import MLPClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Unpacking trained models\n",
    "\n",
    "with open('Pickled data/nn.pickle', 'rb') as f3:\n",
    "    NN = pickle.load(f3)\n",
    "\n",
    "with open('Pickled data/LR.pickle', 'rb') as f3:\n",
    "    LR = pickle.load(f3)\n",
    "\n",
    "with open('Pickled data/naive-bayes.pickle', 'rb') as f3:\n",
    "    NB = pickle.load(f3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Models to train\n",
    "# Neural Network (Single layer with 100 units)\n",
    "# Logistic Regression\n",
    "# Multinomial Naive-Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 1, loss = 0.48831997\n",
      "Iteration 2, loss = 0.47386068\n",
      "Iteration 3, loss = 0.46782249\n",
      "Iteration 4, loss = 0.46039490\n",
      "Iteration 5, loss = 0.45133318\n",
      "Iteration 6, loss = 0.44118853\n",
      "Iteration 7, loss = 0.43009612\n",
      "Iteration 8, loss = 0.41738729\n",
      "Iteration 9, loss = 0.40312573\n",
      "Iteration 10, loss = 0.38755965\n",
      "Iteration 11, loss = 0.37132863\n",
      "Iteration 12, loss = 0.35526305\n",
      "Iteration 13, loss = 0.34024330\n",
      "Iteration 14, loss = 0.32619042\n",
      "Iteration 15, loss = 0.31359916\n",
      "Iteration 16, loss = 0.30232044\n",
      "Iteration 17, loss = 0.29245417\n",
      "Iteration 18, loss = 0.28393783\n",
      "Iteration 19, loss = 0.27643297\n",
      "Iteration 20, loss = 0.26972821\n",
      "Iteration 21, loss = 0.26398571\n",
      "Iteration 22, loss = 0.25880516\n",
      "Iteration 23, loss = 0.25422614\n",
      "Iteration 24, loss = 0.25004071\n",
      "Iteration 25, loss = 0.24643164\n",
      "Iteration 26, loss = 0.24311043\n",
      "Iteration 27, loss = 0.24015910\n",
      "Iteration 28, loss = 0.23728538\n",
      "Iteration 29, loss = 0.23484182\n",
      "Iteration 30, loss = 0.23259847\n",
      "Iteration 31, loss = 0.23025172\n",
      "Iteration 32, loss = 0.22851944\n",
      "Iteration 33, loss = 0.22660175\n",
      "Iteration 34, loss = 0.22511396\n",
      "Iteration 35, loss = 0.22301945\n",
      "Iteration 36, loss = 0.22201668\n",
      "Iteration 37, loss = 0.22054905\n",
      "Iteration 38, loss = 0.21933320\n",
      "Iteration 39, loss = 0.21829570\n",
      "Iteration 40, loss = 0.21687944\n",
      "Iteration 41, loss = 0.21609200\n",
      "Iteration 42, loss = 0.21513536\n",
      "Iteration 43, loss = 0.21417946\n",
      "Iteration 44, loss = 0.21316605\n",
      "Iteration 45, loss = 0.21258028\n",
      "Iteration 46, loss = 0.21172411\n",
      "Iteration 47, loss = 0.21096655\n",
      "Iteration 48, loss = 0.21051166\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/pranav/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/multilayer_perceptron.py:568: UserWarning: Training interrupted by user.\n",
      "  warnings.warn(\"Training interrupted by user.\")\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "MLPClassifier(activation='relu', alpha=0.0001, batch_size='auto', beta_1=0.9,\n",
       "              beta_2=0.999, early_stopping=False, epsilon=1e-08,\n",
       "              hidden_layer_sizes=(100,), learning_rate='constant',\n",
       "              learning_rate_init=0.001, max_iter=200, momentum=0.9,\n",
       "              n_iter_no_change=10, nesterovs_momentum=True, power_t=0.5,\n",
       "              random_state=None, shuffle=True, solver='adam', tol=0.0001,\n",
       "              validation_fraction=0.1, verbose=2, warm_start=False)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Neural Network \n",
    "# (Note - training is suspended after seeing diminishing gain at around the 43rd iteration)\n",
    "\n",
    "NN = MLPClassifier(verbose=2)\n",
    "NN.fit(train_x_transformed, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/pranav/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=100,\n",
       "                   multi_class='warn', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='warn', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Logistic Regression\n",
    "\n",
    "LR = LogisticRegression()\n",
    "LR.fit(train_x_transformed, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Multinomial Naive-Bayes\n",
    "\n",
    "NB = MultinomialNB()\n",
    "NB.fit(train_x_transformed, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Predictions from various models\n",
    "\n",
    "predNN = NN.predict(vector.transform(test_x))\n",
    "predLR = LR.predict(vector.transform(test_x))\n",
    "predNB = NB.predict(vector.transform(test_x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculate accuracy and confusion matrix\n",
    "\n",
    "from sklearn.metrics import confusion_matrix, accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: Neural Network\n",
      "Accuracy - 0.5812875\n",
      "Confusion matrix - [[46587 33319]\n",
      " [33675 46419]]\n",
      "\n",
      "Model: Logistic Regression\n",
      "Accuracy - 0.77554375\n",
      "Confusion matrix - [[59887 20019]\n",
      " [15894 64200]]\n",
      "\n",
      "Model: Naive Bayes\n",
      "Accuracy - 0.76004375\n",
      "Confusion matrix - [[59788 20118]\n",
      " [18275 61819]]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for model, prediction in zip(['Neural Network', 'Logistic Regression', 'Naive Bayes'], [predNN, predLR, predNB]):\n",
    "    print('Model: {}'.format(model))\n",
    "    print('Accuracy - {}'.format(accuracy_score(test_y, prediction)))\n",
    "    print('Confusion matrix - {}\\n'.format(confusion_matrix(test_y, prediction)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Function to test a tweet, defaults to LR due to its higher accuracy\n",
    "\n",
    "def predict(tweet, model = LR):\n",
    "    return model.predict(vector.transform([clean(tweet)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "NN: [1.]\nLR: [1.]\nNB: [1.]\n"
    }
   ],
   "source": [
    "# 1: positive, 0: negative\n",
    "\n",
    "print('NN: {}'.format(predict('I love math!', model = NN)))\n",
    "print('LR: {}'.format(predict('I love math!', model = LR)))\n",
    "print('NB: {}'.format(predict('I love math!', model = NB)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Shelve the model, vector and predict objects\n",
    "\n",
    "import shelve\n",
    "\n",
    "with shelve.open('shelve.model', 'c') as shelf:\n",
    "    shelf['model'] = LR\n",
    "    shelf['vector'] = vector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model 4 (Experimental)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/pranav/anaconda3/envs/TIS/lib/python3.6/site-packages/BertLibrary/bert/optimization.py:87: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n",
      "WARNING:tensorflow:From /Users/pranav/anaconda3/envs/TIS/lib/python3.6/site-packages/BertLibrary/bert_evaluator.py:60: The name tf.train.SessionRunHook is deprecated. Please use tf.estimator.SessionRunHook instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## Using a BERT model to train\n",
    "# Experimenting with BERT, training took over 8 hours on a GPU boosted system. \n",
    "# pip install BertLibrary\n",
    "\n",
    "from BertLibrary import BertFTModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2019-11-11 19:43:11--  https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip\n",
      "Resolving storage.googleapis.com (storage.googleapis.com)... 172.217.8.176\n",
      "Connecting to storage.googleapis.com (storage.googleapis.com)|172.217.8.176|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 407727028 (389M) [application/zip]\n",
      "Saving to: ‘uncased_L-12_H-768_A-12.zip’\n",
      "\n",
      "uncased_L-12_H-768_ 100%[===================>] 388.84M  2.94MB/s    in 2m 11s  \n",
      "\n",
      "2019-11-11 19:45:22 (2.97 MB/s) - ‘uncased_L-12_H-768_A-12.zip’ saved [407727028/407727028]\n",
      "\n",
      "Archive:  uncased_L-12_H-768_A-12.zip\n",
      "   creating: uncased_L-12_H-768_A-12/\n",
      "  inflating: uncased_L-12_H-768_A-12/bert_model.ckpt.meta  \n",
      "  inflating: uncased_L-12_H-768_A-12/bert_model.ckpt.data-00000-of-00001  \n",
      "  inflating: uncased_L-12_H-768_A-12/vocab.txt  \n",
      "  inflating: uncased_L-12_H-768_A-12/bert_model.ckpt.index  \n",
      "  inflating: uncased_L-12_H-768_A-12/bert_config.json  \n"
     ]
    }
   ],
   "source": [
    "# Download and unzip a pre-trained BERT model with 12-layer, 768-hidden, 12-heads, 110M parameters\n",
    "\n",
    "!wget https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip\n",
    "!unzip uncased_L-12_H-768_A-12.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train, df_test = train_test_split(df.drop('Tweet', axis=1), test_size = 0.2, shuffle = True)\n",
    "df_dev, df_dev2 = train_test_split(df_test, test_size=0.9, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Score</th>\n",
       "      <th>TweetStripped</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>442917</th>\n",
       "      <td>0.0</td>\n",
       "      <td>miss church today... need prayer &amp;amp; encoura...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>239920</th>\n",
       "      <td>0.0</td>\n",
       "      <td>sun shine i'm tire enjoy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>911843</th>\n",
       "      <td>1.0</td>\n",
       "      <td>long weekend. thank god sleep monday</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>316239</th>\n",
       "      <td>0.0</td>\n",
       "      <td>enjoy time off...back work</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>520169</th>\n",
       "      <td>0.0</td>\n",
       "      <td>actual spot leed</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Score                                      TweetStripped\n",
       "442917    0.0  miss church today... need prayer &amp; encoura...\n",
       "239920    0.0                           sun shine i'm tire enjoy\n",
       "911843    1.0               long weekend. thank god sleep monday\n",
       "316239    0.0                         enjoy time off...back work\n",
       "520169    0.0                                   actual spot leed"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_dev.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the dataset for the BERT model\n",
    "\n",
    "!mkdir dataset\n",
    "\n",
    "df_train.reset_index(drop=True).to_csv('dataset/train.csv', sep='\\t', index=None, header=None)\n",
    "df_test.to_csv('dataset/test.csv', sep='\\t', index=None, header=None)\n",
    "df_dev.to_csv('dataset/dev.csv', sep='\\t', index=None, header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/pranav/anaconda3/envs/TIS/lib/python3.6/site-packages/BertLibrary/bert/modeling.py:93: The name tf.gfile.GFile is deprecated. Please use tf.io.gfile.GFile instead.\n",
      "\n",
      "WARNING:tensorflow:From /Users/pranav/anaconda3/envs/TIS/lib/python3.6/site-packages/BertLibrary/models/BertModel.py:63: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
      "\n",
      "INFO:tensorflow:Using config: {'_model_dir': 'output', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 500, '_save_checkpoints_secs': None, '_session_config': device_count {\n",
      "  key: \"GPU\"\n",
      "  value: 1\n",
      "}\n",
      "gpu_options {\n",
      "  per_process_gpu_memory_fraction: 0.5\n",
      "  allow_growth: true\n",
      "}\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fe960ea1ef0>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n"
     ]
    }
   ],
   "source": [
    "bert_model = BertFTModel(model_dir='uncased_L-12_H-768_A-12',\n",
    "                        ckpt_name='bert_model.ckpt',\n",
    "                        labels = ['0', '1'],\n",
    "                        ckpt_output_dir='output',\n",
    "                        num_train_steps=20000,\n",
    "                        num_warmup_steps=500,\n",
    "                        save_check_steps=500,\n",
    "                        do_lower_case=False,\n",
    "                        max_seq_len=50,\n",
    "                        batch_size=32)\n",
    "\n",
    "bert_trainer = bert_model.get_trainer()\n",
    "bert_evaluator = bert_model.get_evaluator()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_SIZE = 0.75\n",
    "VAL_SIZE = 0.05\n",
    "dataset_count = len(df)\n",
    "\n",
    "df_train_val, df_test = train_test_split(temp, test_size=1-TRAIN_SIZE-VAL_SIZE, random_state=42)\n",
    "df_train, df_val = train_test_split(df_train_val, test_size=VAL_SIZE / (VAL_SIZE + TRAIN_SIZE), random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = df.drop('Tweet', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp['Score'] = temp['Score'].apply(lambda x : int(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Score</th>\n",
       "      <th>TweetStripped</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>- awww, that' bummer. shoulda got david carr t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>upset can't updat facebook text it... might cr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>dive mani time ball. manag save 50% rest go bound</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>whole bodi feel itchi like fire</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>no, behav all. i'm mad. here? can't see there.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Score                                      TweetStripped\n",
       "0      0  - awww, that' bummer. shoulda got david carr t...\n",
       "1      0  upset can't updat facebook text it... might cr...\n",
       "2      0  dive mani time ball. manag save 50% rest go bound\n",
       "3      0                    whole bodi feel itchi like fire\n",
       "4      0     no, behav all. i'm mad. here? can't see there."
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir dataset\n",
    "\n",
    "df_train.sample(frac=1.0).reset_index(drop=True).to_csv('dataset/train.tsv', sep='\\t', index=None, header=None)\n",
    "df_val.to_csv('dataset/dev.tsv', sep='\\t', index=None, header=None)\n",
    "df_test.to_csv('dataset/test.tsv', sep='\\t', index=None, header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "#bert_trainer.train_from_file('dataset', 35000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ft_evaluator.evaluate_from_file('dataset', checkpoint=\"output/model.ckpt-35000\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = predict('Ad')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a[0] == 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}