ch08 - Sentiment analysis (NLP and big data).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pyprind\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "import nltk\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.feature_extraction.text import HashingVectorizer\n",
    "from sklearn.linear_model import SGDClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# dataset http://ai.stanford.edu/~amaas/data/sentiment\n",
    "# change the `basepath` to the directory of the\n",
    "# unzipped movie dataset \n",
    "\n",
    "basepath = './aclImdb'\n",
    "\n",
    "labels = {'pos': 1, 'neg': 0}\n",
    "pbar = pyprind.ProgBar(50000)\n",
    "df = pd.DataFrame()\n",
    "for s in ('test', 'train'):\n",
    "    for l in ('pos', 'neg'):\n",
    "        path = os.path.join(basepath, s, l)\n",
    "        for file in os.listdir(path):\n",
    "            with open(os.path.join(path, file), 'r', encoding='utf-8') as infile:\n",
    "                txt = infile.read()\n",
    "            df = df.append([[txt, labels[l]]], ignore_index=True)\n",
    "            pbar.update()\n",
    "df.columns = ['review', 'sentiment']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.random.seed(0)\n",
    "df = df.reindex(np.random.permutation(df.index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.to_csv('./movie_data.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>In 1974, the teenager Martha Moxley (Maggie Gr...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>OK... so... I really like Kris Kristofferson a...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>***SPOILER*** Do not read this, if you think a...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              review  sentiment\n",
       "0  In 1974, the teenager Martha Moxley (Maggie Gr...          1\n",
       "1  OK... so... I really like Kris Kristofferson a...          0\n",
       "2  ***SPOILER*** Do not read this, if you think a...          0"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('./movie_data.csv')\n",
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Bag-of-words model\n",
    "# Transforming documents into feature vectors\n",
    "count = CountVectorizer()\n",
    "docs = np.array([\n",
    "        'The sun is shining',\n",
    "        'The weather is sweet',\n",
    "        'The sun is shining, the weather is sweet, and one and one is two'])\n",
    "bag = count.fit_transform(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'the': 6, 'is': 1, 'weather': 8, 'one': 2, 'sun': 4, 'sweet': 5, 'shining': 3, 'and': 0, 'two': 7}\n"
     ]
    }
   ],
   "source": [
    "print(count.vocabulary_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 1 0 1 1 0 1 0 0]\n",
      " [0 1 0 0 0 1 1 0 1]\n",
      " [2 3 2 1 1 1 2 1 1]]\n"
     ]
    }
   ],
   "source": [
    "print(bag.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.    0.43  0.    0.56  0.56  0.    0.43  0.    0.  ]\n",
      " [ 0.    0.43  0.    0.    0.    0.56  0.43  0.    0.56]\n",
      " [ 0.5   0.45  0.5   0.19  0.19  0.19  0.3   0.25  0.19]]\n"
     ]
    }
   ],
   "source": [
    "# Assessing word relevancy via term frequency-inverse document frequency\n",
    "np.set_printoptions(precision=2)\n",
    "tfidf = TfidfTransformer()\n",
    "print(tfidf.fit_transform(count.fit_transform(docs)).toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# manual example with digit\n",
    "tf_is = 3\n",
    "n_docs = 3\n",
    "idf_is = np.log((n_docs+1) / (3+1))\n",
    "tfidf_is = tf_is * (idf_is + 1)\n",
    "# then apply L2 normalization ( v_L2 = v/norm(v) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 3.39,  3.  ,  3.39,  1.29,  1.29,  1.29,  2.  ,  1.69,  1.29])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# example with vector\n",
    "tfidf = TfidfTransformer(use_idf=True, norm=None, smooth_idf=True)\n",
    "raw_tfidf = tfidf.fit_transform(count.fit_transform(docs)).toarray()[-1]\n",
    "raw_tfidf "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.5 ,  0.45,  0.5 ,  0.19,  0.19,  0.19,  0.3 ,  0.25,  0.19])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l2_tfidf = raw_tfidf / np.sqrt(np.sum(raw_tfidf**2))\n",
    "l2_tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "is seven.<br /><br />Title (Brazil): Not Available\n"
     ]
    }
   ],
   "source": [
    "# Cleaning text data\n",
    "print(df.loc[0, 'review'][-50:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import re\n",
    "def preprocessor(text):\n",
    "    text = re.sub('<[^>]*>', '', text)\n",
    "    emoticons = re.findall('(?::|;|=)(?:-)?(?:\\)|\\(|D|P)', text)\n",
    "    text = re.sub('[\\W]+', ' ', text.lower()) +\\\n",
    "        ' '.join(emoticons).replace('-', '')\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'is seven title brazil not available'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check whether the function works\n",
    "preprocessor(df.loc[0, 'review'][-50:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'this is a test :) :( :)'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor(\"</a>This :) is :( a test :-)!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['review'] = df['review'].apply(preprocessor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Processing documents into tokens\n",
    "porter = PorterStemmer()\n",
    "\n",
    "def tokenizer(text):\n",
    "    return text.split()\n",
    "\n",
    "\n",
    "def tokenizer_porter(text):\n",
    "    return [porter.stem(word) for word in text.split()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['runners', 'like', 'running', 'and', 'thus', 'they', 'run']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer('runners like running and thus they run')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['runner', 'like', 'run', 'and', 'thu', 'they', 'run']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer_porter('runners like running and thus they run')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\it\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# how to delete common words aka stopwords\n",
    "# fist download the list of stopwords for english\n",
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['runner', 'like', 'run', 'run', 'lot']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# example of deleting stopwords\n",
    "stop = stopwords.words('english')\n",
    "[w for w in tokenizer_porter('a runner likes running and runs a lot')[-10:]\n",
    "if w not in stop]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Training a logistic regression model for document classification\n",
    "X_train = df.loc[:25000, 'review'].values\n",
    "y_train = df.loc[:25000, 'sentiment'].values\n",
    "X_test = df.loc[25000:, 'review'].values\n",
    "y_test = df.loc[25000:, 'sentiment'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tfidf = TfidfVectorizer(strip_accents=None,\n",
    "                        lowercase=False,\n",
    "                        preprocessor=None)\n",
    "\n",
    "param_grid = [{'vect__ngram_range': [(1, 1)],\n",
    "               'vect__stop_words': [stop, None],\n",
    "               'vect__tokenizer': [tokenizer, tokenizer_porter],\n",
    "               'clf__penalty': ['l1', 'l2'],\n",
    "               'clf__C': [1.0, 10.0, 100.0]},\n",
    "              {'vect__ngram_range': [(1, 1)],\n",
    "               'vect__stop_words': [stop, None],\n",
    "               'vect__tokenizer': [tokenizer, tokenizer_porter],\n",
    "               'vect__use_idf':[False],\n",
    "               'vect__norm':[None],\n",
    "               'clf__penalty': ['l1', 'l2'],\n",
    "               'clf__C': [1.0, 10.0, 100.0]},\n",
    "              ]\n",
    "\n",
    "lr_tfidf = Pipeline([('vect', tfidf),\n",
    "                     ('clf', LogisticRegression(random_state=0))])\n",
    "\n",
    "gs_lr_tfidf = GridSearchCV(lr_tfidf, param_grid,\n",
    "                           scoring='accuracy',\n",
    "                           cv=5,\n",
    "                           verbose=1,\n",
    "                           n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "gs_lr_tfidf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print('Best parameter set: %s ' % gs_lr_tfidf.best_params_)\n",
    "print('CV Accuracy: %.3f' % gs_lr_tfidf.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# average score on grid search\n",
    "clf = gs_lr_tfidf.best_estimator_\n",
    "print('Test Accuracy: %.3f' % clf.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Working with bigger data - online algorithms and out-of-core learning\n",
    "def tokenizer(text):\n",
    "    text = re.sub('<[^>]*>', '', text)\n",
    "    emoticons = re.findall('(?::|;|=)(?:-)?(?:\\)|\\(|D|P)', text.lower())\n",
    "    text = re.sub('[\\W]+', ' ', text.lower()) +\\\n",
    "        ' '.join(emoticons).replace('-', '')\n",
    "    tokenized = [w for w in text.split() if w not in stop]\n",
    "    return tokenized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def stream_docs(path):\n",
    "    with open(path, 'r', encoding='utf-8') as csv:\n",
    "        next(csv)  # skip header\n",
    "        for line in csv:\n",
    "            text, label = line[:-3], int(line[-2])\n",
    "            yield text, label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('\"In 1974, the teenager Martha Moxley (Maggie Grace) moves to the high-class area of Belle Haven, Greenwich, Connecticut. On the Mischief Night, eve of Halloween, she was murdered in the backyard of her house and her murder remained unsolved. Twenty-two years later, the writer Mark Fuhrman (Christopher Meloni), who is a former LA detective that has fallen in disgrace for perjury in O.J. Simpson trial and moved to Idaho, decides to investigate the case with his partner Stephen Weeks (Andrew Mitchell) with the purpose of writing a book. The locals squirm and do not welcome them, but with the support of the retired detective Steve Carroll (Robert Forster) that was in charge of the investigation in the 70\\'s, they discover the criminal and a net of power and money to cover the murder.<br /><br />\"\"Murder in Greenwich\"\" is a good TV movie, with the true story of a murder of a fifteen years old girl that was committed by a wealthy teenager whose mother was a Kennedy. The powerful and rich family used their influence to cover the murder for more than twenty years. However, a snoopy detective and convicted perjurer in disgrace was able to disclose how the hideous crime was committed. The screenplay shows the investigation of Mark and the last days of Martha in parallel, but there is a lack of the emotion in the dramatization. My vote is seven.<br /><br />Title (Brazil): Not Available\"',\n",
       " 1)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next(stream_docs(path='./movie_data.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_minibatch(doc_stream, size):\n",
    "    docs, y = [], []\n",
    "    try:\n",
    "        for _ in range(size):\n",
    "            text, label = next(doc_stream)\n",
    "            docs.append(text)\n",
    "            y.append(label)\n",
    "    except StopIteration:\n",
    "        return None, None\n",
    "    return docs, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vect = HashingVectorizer(decode_error='ignore', \n",
    "                         n_features=2**21,\n",
    "                         preprocessor=None, \n",
    "                         tokenizer=tokenizer)\n",
    "\n",
    "clf = SGDClassifier(loss='log', random_state=1, n_iter=1)\n",
    "doc_stream = stream_docs(path='./movie_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [##############################] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:54\n"
     ]
    }
   ],
   "source": [
    "pbar = pyprind.ProgBar(45)\n",
    "\n",
    "classes = np.array([0, 1])\n",
    "for _ in range(45):\n",
    "    X_train, y_train = get_minibatch(doc_stream, size=1000)\n",
    "    if not X_train:\n",
    "        break\n",
    "    X_train = vect.transform(X_train)\n",
    "    clf.partial_fit(X_train, y_train, classes=classes)\n",
    "    pbar.update()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.867\n"
     ]
    }
   ],
   "source": [
    "X_test, y_test = get_minibatch(doc_stream, size=5000)\n",
    "X_test = vect.transform(X_test)\n",
    "print('Accuracy: %.3f' % clf.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# update the model with new data\n",
    "clf = clf.partial_fit(X_test, y_test)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}