single process yelp.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from time import perf_counter\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\ivy_x\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('wordnet')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Single Process"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## load in file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished in 0.37 sec\n"
     ]
    }
   ],
   "source": [
    "start=perf_counter()\n",
    "review20=pd.read_json(\"sampled30k.json\",lines=True)\n",
    "elapsed_time=perf_counter()-start\n",
    "print(f'Finished in {elapsed_time:.2f} sec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stars</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>I went here today to upgrade to an iPhone X. T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>I went here yesterday and I got the best servi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>These are sooo good!  I'm having Bulgogi dog c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>First time buyer,\\nI was excited to go in buy ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Oh my gosh.. BOMBBBBBBBB!!! This place is so a...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   stars                                               text\n",
       "0      5  I went here today to upgrade to an iPhone X. T...\n",
       "1      5  I went here yesterday and I got the best servi...\n",
       "2      5  These are sooo good!  I'm having Bulgogi dog c...\n",
       "3      1  First time buyer,\\nI was excited to go in buy ...\n",
       "4      5  Oh my gosh.. BOMBBBBBBBB!!! This place is so a..."
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "review20.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(30163, 2)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "review20.shape\n",
    "# 360965 90%\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "review20['sentiment'] = np.where(review20['stars']<5, 0, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "review20.drop(columns=['stars'],inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## pre-processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove stop words\n",
    "# lemmatize\n",
    "import re\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "\n",
    "stop_words = ['ourselves', 'hers', 'between', 'yourself', 'but', 'again', 'there', 'about', 'once', 'during', 'out', 'very', 'having', 'with', 'they', 'own', 'an', 'be', 'some', 'for', 'do', 'its', 'yours', 'such', 'into', 'of', 'most', 'itself', 'other', 'off', 'is', 's', 'am', 'or', 'who', 'as', 'from', 'him', 'each', 'the', 'themselves', 'until', 'below', 'are', 'we', 'these', 'your', 'his', 'through', 'don', 'nor', 'me', 'were', 'her', 'more', 'himself', 'this', 'down', 'should', 'our', 'their', 'while', 'above', 'both', 'up', 'to', 'ours', 'had', 'she', 'all', 'no', 'when', 'at', 'any', 'before', 'them', 'same', 'and', 'been', 'have', 'in', 'will', 'on', 'does', 'yourselves', 'then', 'that', 'because', 'what', 'over', 'why', 'so', 'can', 'did', 'not', 'now', 'under', 'he', 'you', 'herself', 'has', 'just', 'where', 'too', 'only', 'myself', 'which', 'those', 'i', 'after', 'few', 'whom', 't', 'being', 'if', 'theirs', 'my', 'against', 'a', 'by', 'doing', 'it', 'how', 'further', 'was', 'here', 'than']\n",
    "\n",
    "\n",
    "def normalize(text):\n",
    "    tokens = re.findall('[a-zA-Z\\']+', text)\n",
    "    tokens_without_stop_words = [lemmatizer.lemmatize(i.lower(), pos='v') for i in tokens if i.lower() not in stop_words]\n",
    "    return tokens_without_stop_words\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sample_size=30000\n",
    "# review20_sampled=review20.truncate(before=0,after=sample_size-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_size=30000\n",
    "review20_sampled=review20.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I went here today to upgrade to an iPhone X. T...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>I went here yesterday and I got the best servi...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>These are sooo good!  I'm having Bulgogi dog c...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>First time buyer,\\nI was excited to go in buy ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Oh my gosh.. BOMBBBBBBBB!!! This place is so a...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  sentiment\n",
       "0  I went here today to upgrade to an iPhone X. T...          1\n",
       "1  I went here yesterday and I got the best servi...          1\n",
       "2  These are sooo good!  I'm having Bulgogi dog c...          1\n",
       "3  First time buyer,\\nI was excited to go in buy ...          0\n",
       "4  Oh my gosh.. BOMBBBBBBBB!!! This place is so a...          1"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "review20_sampled.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pre_process(data):\n",
    "    data.text=data.text.apply(lambda x: normalize(x))\n",
    "    return(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished in 15.28 sec\n"
     ]
    }
   ],
   "source": [
    "start=perf_counter()\n",
    "# review20.text=review20.text.apply(lambda x: normalize(x))\n",
    "review20_sampled=pre_process(review20_sampled)\n",
    "elapsed_time=perf_counter()-start\n",
    "print(f'Finished in {elapsed_time:.2f} sec')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def document_features(document):\n",
    "    document_words = set(document)\n",
    "    features = {}\n",
    "    for word in word_features:\n",
    "        features['contains({})'.format(word)] = (word in document_words)\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finish truncating\n",
      "finish feature engineering\n",
      "finish data split\n",
      "finish training\n",
      "Finished in 158.84 sec\n"
     ]
    }
   ],
   "source": [
    "# from nltk.corpus import movie_reviews\n",
    "\n",
    "from itertools import chain\n",
    "from math import floor\n",
    "\n",
    "start=perf_counter()\n",
    "\n",
    "documents=[tuple(x) for x in review20_sampled.values]\n",
    "\n",
    "print('finish truncating')\n",
    "\n",
    "all_word=review20_sampled.text.values.tolist()\n",
    "\n",
    "all_text_list=list(chain(*all_word))\n",
    "\n",
    "all_words = nltk.FreqDist(all_text_list)\n",
    "\n",
    "word_features = list(all_words)[:2000]\n",
    "\n",
    "featuresets = [(document_features(d), c) for (d,c) in documents]\n",
    "print(\"finish feature engineering\")\n",
    "train_set, test_set = featuresets[:floor(0.9*sample_size)], featuresets[floor(0.9*sample_size):]\n",
    "print('finish data split')\n",
    "classifier = nltk.NaiveBayesClassifier.train(train_set)\n",
    "print('finish training')\n",
    "elapsed_time=perf_counter()-start\n",
    "print(f'Finished in {elapsed_time:.2f} sec')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7944988934555801\n",
      "Finished in 24.24 sec\n"
     ]
    }
   ],
   "source": [
    "# Test the classifier\n",
    "start=perf_counter()\n",
    "print(nltk.classify.accuracy(classifier, test_set))\n",
    "elapsed_time=perf_counter()-start\n",
    "print(f'Finished in {elapsed_time:.2f} sec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}