main.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IMDB Sentiment Analysis\n",
    "<strong>Adam El Kholy</strong> \\\n",
    "<strong>University of Bath</strong> \\\n",
    "Last Updated: <strong>06/12/2023</strong>\n",
    "\n",
    "Free to use under the Apache 2.0 license \\\n",
    "For use with the [IMDB reviews](https://www.kaggle.com/datasets/lakshmi25npathi/imdb-dataset-of-50k-movie-reviews) dataset available on Kaggle \n",
    "\n",
    "The following notebook allows you to train and evaluate the following models for the task of sentiment analysis using on the IMDB movie dataset\n",
    "- Multinomial Naive Bayes (manual implementation)\n",
    "- Gaussian Naive Bayes (manual implementation)\n",
    "- Sklearn MNB\n",
    "- Sklearn GNB\n",
    "- Logistic Regression\n",
    "- Support Vector Machines\n",
    "\n",
    "See ```BERT.ipynb``` for the evaluation of BERT (cased and uncased) on the same task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import nltk\n",
    "import string\n",
    "import numpy as np\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.lancaster import LancasterStemmer\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "\n",
    "nltk.download('stopwords')\n",
    "nltk.download('punkt')\n",
    "nltk.download('wordnet')\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Directory structure: \n",
    "data/ \n",
    "    pos/\n",
    "        1.txt\n",
    "        2.txt\n",
    "        ...\n",
    "    neg/\n",
    "        1.txt\n",
    "        2.txt\n",
    "        ...\n",
    "where pos/ and neg/ contain positive and negative reviews respectively\n",
    "\"\"\"\n",
    "def read_corpus(directory):\n",
    "    files = [f for f in os.listdir(directory) if f.endswith('.txt')]\n",
    "    corpus = []\n",
    "    for file in files:\n",
    "        with open(os.path.join(directory, file), 'r', encoding='utf-8') as f:\n",
    "            document = f.read()\n",
    "            corpus.append(document)\n",
    "    return corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Homelessness (or Houselessness as George Carlin stated) has been an issue for years but never a plan to help those on the street ...\n",
      "This film is about a male escort getting involved in a murder investigation that happened in the circle of powerful men's wives. ...\n"
     ]
    }
   ],
   "source": [
    "# unpack data into corpus vars\n",
    "positive_corpus = read_corpus(\"data/pos/\")\n",
    "negative_corpus = read_corpus(\"data/neg/\")\n",
    "corpus = positive_corpus + negative_corpus\n",
    "positive_labels = len(positive_corpus)\n",
    "negative_labels = len(negative_corpus)\n",
    "corpus_length = len(corpus)\n",
    "\n",
    "# sanity check\n",
    "print(positive_corpus[0][:128])\n",
    "print(negative_corpus[1][:128])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Splitting the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2720, 29379)\n",
      "(800, 29379)\n",
      "(480, 29379)\n",
      "[0.        0.0105175 0.        ... 0.        0.        0.       ]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "r_seed = 42\n",
    "\n",
    "def get_test_train_dev_split(X):\n",
    "    y = np.concatenate([np.ones(positive_labels), np.zeros(negative_labels)])\n",
    "\n",
    "    # we first split the data into train and test\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.80, \n",
    "                                                        shuffle=True, random_state=r_seed)\n",
    "    \n",
    "    # then split train into train and development set \n",
    "    X_train, X_dev, y_train, y_dev = train_test_split(X_train, y_train, \n",
    "                                                      test_size=0.15, \n",
    "                                                      shuffle=True, random_state=r_seed)\n",
    "   \n",
    "    #68% train, 12% validation, 20% test\n",
    "    return X_train, y_train, X_test, y_test, X_dev, y_dev\n",
    "\n",
    "X_train, y_train, X_test, y_test, X_dev, y_dev = get_test_train_dev_split(stem_unigram_tf_idf)\n",
    "print(X_train.shape)\n",
    "print(X_test.shape)\n",
    "print(X_dev.shape)\n",
    "print(X_train.toarray()[0])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Generation Using N-Grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk import word_tokenize\n",
    "from nltk import ngrams\n",
    "\n",
    "def text_to_ngrams(sentence, n, remove_stopwords=True):\n",
    "    # set of stopwords to remove\n",
    "    stoplist = set(stopwords.words('english')) \n",
    "    if not remove_stopwords:\n",
    "        stoplist = set()\n",
    "    tokenised_words = [word for word in word_tokenize(sentence.lower()) \n",
    "                       if word not in stoplist and word not in string.punctuation and word != \"br\"] \n",
    "                       # a list of tokenised words with stop-words, punctuation, and <br>s removed\n",
    "    # apply nltk n-grams algorithm\n",
    "    zipped_grams = ngrams(tokenised_words, n) \n",
    "    return list(zipped_grams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('ozymandias', 'king', 'kings'), ('king', 'kings', 'look'), ('kings', 'look', 'upon'), ('look', 'upon', 'works'), ('upon', 'works', 'ye'), ('works', 'ye', 'mighty'), ('ye', 'mighty', 'despair')]\n",
      "('ozymandias', 'king', 'kings')\n",
      "('king', 'kings', 'look')\n",
      "('kings', 'look', 'upon')\n",
      "('look', 'upon', 'works')\n",
      "('upon', 'works', 'ye')\n",
      "('works', 'ye', 'mighty')\n",
      "('ye', 'mighty', 'despair')\n"
     ]
    }
   ],
   "source": [
    "sentence = \"I am Ozymandias, king of kings, look upon my works ye mighty and despair\"\n",
    "grams = text_to_ngrams(sentence, 3)\n",
    "print(grams)\n",
    "for gram in grams:\n",
    "    print(gram)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert the entire corpus to ngrams\n",
    "def corpus_to_ngrams(corpus, n, remove_stopwords=True):\n",
    "    new_corpus = []\n",
    "    for text in corpus:\n",
    "        new_corpus.append(text_to_ngrams(text, n, remove_stopwords))\n",
    "    return new_corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('homelessness',), ('houselessness',), ('george',), ('carlin',), ('stated',), ('issue',), ('years',), ('never',), ('plan',), ('help',), ('street',) ... ]\n"
     ]
    }
   ],
   "source": [
    "corpus_unigrams = corpus_to_ngrams(corpus, 1)\n",
    "print(corpus_unigrams[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('homelessness', 'houselessness'), ('houselessness', 'george'), ('george', 'carlin'), ('carlin', 'stated'), ('stated', 'issue'), ('issue', 'years'), ('years', 'never'), ('never', 'plan'), ('plan', 'help'), ('help', 'street') ... ]\n"
     ]
    }
   ],
   "source": [
    "corpus_bigrams = corpus_to_ngrams(corpus, 2)\n",
    "print(corpus_bigrams[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('homelessness', 'houselessness', 'george'), ('houselessness', 'george', 'carlin'), ('george', 'carlin', 'stated'), ('carlin', 'stated', 'issue'), ('stated', 'issue', 'years'), ('issue', 'years', 'never'), ('years', 'never', 'plan'), ('never', 'plan', 'help'), ('plan', 'help', 'street') ... \n"
     ]
    }
   ],
   "source": [
    "corpus_trigrams = corpus_to_ngrams(corpus, 3)\n",
    "print(corpus_trigrams[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('homelessness',), ('or',), ('houselessness',), ('as',), ('george',), ('carlin',), ('stated',), ('has',), ('been',), ('an',), ('issue',), ('for',), ('years',), ('but',), ('never',), ('a',), ('plan',), ('to',), ('help',), ('those',), ('on',), ('the',), ('street',) ...]\n"
     ]
    }
   ],
   "source": [
    "# with stopwords included\n",
    "corps_unigrams_with_stopwords = corpus_to_ngrams(corpus, 1, remove_stopwords=False)\n",
    "print(corps_unigrams_with_stopwords[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Selection using Lemmatization and Stemming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def apply_stemming(text):\n",
    "    st = LancasterStemmer()\n",
    "    word_list = [\" \".join(st.stem(gram) for gram in ngram) for ngram in text]\n",
    "                # stems the list of ngram tuples using nltk's LancasterStemmer\n",
    "    return word_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ozymandia king king\n",
      "king king look\n",
      "king look upon\n",
      "look upon work\n",
      "upon work ye\n",
      "work ye mighty\n",
      "ye mighty despair\n"
     ]
    }
   ],
   "source": [
    "stemmed_text = apply_stemming(grams)\n",
    "for feature in stemmed_text:\n",
    "    print(feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def apply_lemmatization(text):\n",
    "    lm = WordNetLemmatizer()\n",
    "    word_list = [\" \".join(lm.lemmatize(gram) for gram in ngram) for ngram in text]\n",
    "                # lemmatizes the list of ngram tuples\n",
    "    return word_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ozymandias king king\n",
      "king king look\n",
      "king look upon\n",
      "look upon work\n",
      "upon work ye\n",
      "work ye mighty\n",
      "ye mighty despair\n"
     ]
    }
   ],
   "source": [
    "lemmatized_text = apply_lemmatization(grams)\n",
    "for feature in lemmatized_text:\n",
    "    print(feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "# apply a given stemming or lemmatization function to the corpus\n",
    "def apply_to_corpus(func, corpus):\n",
    "    new_corpus = []\n",
    "    for text in corpus:\n",
    "        new_corpus.append(func(text))\n",
    "    return new_corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['homelessness', 'houselessness', 'george', 'carlin', 'stated', 'issue', 'year', 'never', 'plan', 'help']\n",
      "['homeless', 'houseless', 'georg', 'carlin', 'stat', 'issu', 'year', 'nev', 'plan', 'help']\n"
     ]
    }
   ],
   "source": [
    "lemmatized_unigrams = apply_to_corpus(apply_lemmatization, corpus_unigrams)\n",
    "stemmed_unigrams = apply_to_corpus(apply_stemming, corpus_unigrams)\n",
    "print(lemmatized_unigrams[0][:10])\n",
    "print(stemmed_unigrams[0][:10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TF-IDF Feature Extraction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First we set about generating a shared vocabulary, containing the number of documents each unique word occurs in, so as to calculate TF and IDF values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_shared_vocabulary(corpus):\n",
    "    words = {}\n",
    "    for text in corpus:\n",
    "        for word in set(text): \n",
    "            # set(text) removes duplicates, meaning the dictionary contains document frequency values \n",
    "            if word in words:\n",
    "                words[word] += 1\n",
    "            else:\n",
    "                words[word] = 1\n",
    "    return words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "shared_vocabulary = generate_shared_vocabulary(stemmed_unigrams)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we generate our TF-IDF matrix, where each row represents a document in the corpus. We utilise the scipy sparse matrix data structure in order to save memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy import sparse\n",
    "\n",
    "def generate_tf_idf_matrix(corpus, shared_vocabulary, one_hot=False):\n",
    "    N = len(shared_vocabulary)\n",
    "    shared_vocabulary_list = list(shared_vocabulary)\n",
    "    matrix = sparse.lil_matrix(np.zeros([corpus_length, N]))\n",
    "    # sparse list of lists to store our tf_idf values\n",
    "\n",
    "    for i, text in enumerate(corpus):\n",
    "        # calculate tf_idf for each feature in each document and insert in correct index\n",
    "        for word in text: \n",
    "            index = shared_vocabulary_list.index(word)\n",
    "            tf = text.count(word) / len(text)\n",
    "            idf = np.log10(N / (shared_vocabulary[word] +1))\n",
    "            # if using one_hot vectors for SVM and LogReg then simply insert 1\n",
    "            matrix[i, index] = tf * idf if not one_hot else 1\n",
    "    return matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.01005802, 0.00693199, 0.01359507, ..., 0.        , 0.        ,\n",
       "        0.        ]])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stem_unigram_tf_idf = generate_tf_idf_matrix(stemmed_unigrams, shared_vocabulary)\n",
    "stem_unigram_tf_idf[0].toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Multinomial Naïve Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Calculates the likelihood that a feature x belongs to class C, p(x|C)\n",
    "    labels: 1 for the class whose likelihood is being calculated, 0 for any others\n",
    "    data: training data, i.e. our TF-IDF matrix of all documents\n",
    "    alpha: value for laplace smoothing\n",
    "\"\"\"\n",
    "def calculate_likelihoods(data, labels, alpha=1.0):\n",
    "    N = data.shape[1]\n",
    "    likelihoods = np.zeros([N])\n",
    "    for i in range(N):\n",
    "        feature = data[:, i].toarray().flatten()\n",
    "        likelihoods[i] = (np.sum(feature * labels) + alpha)  / (np.sum(labels) + alpha) \n",
    "        # likelihood calculation (p(X|C)) using laplace smoothing\n",
    "    return likelihoods\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Calculates the likelihoods for both classes given the training data as well as priors for both classes\n",
    "\"\"\"\n",
    "def train_multinomial_bayes(X_train, y_train):\n",
    "    # inverted label array, denoting 1 for the negative class and 0 for the positive, used for calculating likelihood and priors\n",
    "    inverted_y_train = np.array([not y for y in y_train]).astype(int)\n",
    "    \n",
    "    pos_likelihoods = calculate_likelihoods(X_train, y_train)\n",
    "    neg_likelihoods = calculate_likelihoods(X_train, inverted_y_train)\n",
    "    pos_log_likelihoods = np.log(pos_likelihoods)\n",
    "    neg_log_likelihoods = np.log(neg_likelihoods)\n",
    "\n",
    "    pos_prior = np.sum(y_train) / len(y_train)\n",
    "    neg_prior = np.sum(inverted_y_train) / len(inverted_y_train)\n",
    "    pos_log_prior = np.log(pos_prior)\n",
    "    neg_log_prior = np.log(neg_prior)\n",
    "    return pos_log_likelihoods, neg_log_likelihoods, pos_log_prior, neg_log_prior\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Assigns a class label to a given document using likelihoods and priors\n",
    "\"\"\"\n",
    "def get_multinomial_class_label(data, document):\n",
    "    pos_log_likelihoods, neg_log_likelihoods, pos_log_prior, neg_log_prior = data\n",
    "\n",
    "    # unpack our sparse vector:\n",
    "    features = np.nonzero(document)[0] \n",
    "    pos_total = 0\n",
    "    neg_total = 0\n",
    "\n",
    "    for index in features: \n",
    "        # sum log likelihoods for each feature, for both classes\n",
    "        pos_total += pos_log_likelihoods[index]\n",
    "        neg_total += neg_log_likelihoods[index]\n",
    "        \n",
    "    # add priors\n",
    "    pos_total += pos_log_prior \n",
    "    neg_total += neg_log_prior\n",
    "    class_label = 1 if pos_total > neg_total else 0 \n",
    "    return class_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Runs the entire pipeline for MNB and returns a predictions array\n",
    "\"\"\"\n",
    "def test_train_multinomial_bayes(train_data, train_labels, test_data):\n",
    "    data = train_multinomial_bayes(train_data, train_labels)\n",
    "    predictions = []\n",
    "    for i, _ in enumerate(test_data):\n",
    "        doc =  test_data[i].toarray().flatten()\n",
    "        label = get_multinomial_class_label(data, doc)\n",
    "        predictions.append(label)\n",
    "    return predictions\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "# outputs full model evaluation\n",
    "def evaluate_model(test_labels, predictions):\n",
    "    print(f\"accuracy: {accuracy_score(test_labels, predictions)}\")\n",
    "    print(f\"precision: {precision_score(test_labels, predictions)}\")\n",
    "    print(f\"recall: {recall_score(test_labels, predictions)}\")\n",
    "    print(f\"f1 score: {f1_score(test_labels, predictions)}\")\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluation for Multinomial Naive Bayes on the development set for stemmed unigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.875\n",
      "precision: 0.9267015706806283\n",
      "recall: 0.7937219730941704\n",
      "f1 score: 0.8550724637681159\n"
     ]
    }
   ],
   "source": [
    "predictions = test_train_multinomial_bayes(X_train, y_train, X_dev)\n",
    "evaluate_model(y_dev, predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gaussian Naïve Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Calculates the mean and standard distribution for a given feature using TF-IDF scores across all documents\n",
    "    labels: 1 for the class whose likelihood is being calculated, 0 for any others\n",
    "    data: training data, i.e. our TF-IDF matrix of all documents\n",
    "    alpha: value for laplace smoothing\n",
    "\"\"\"\n",
    "def calculate_guassian_distributions(data, labels, alpha=1e-10):\n",
    "    pos_distribution = []\n",
    "    neg_distribution = []\n",
    "    inverted_labels = np.array([not y for y in labels]).astype(int)\n",
    "\n",
    "    # calculate means and standard deviations for each feature in order to compute distributions\n",
    "    for i in range(data.shape[1]):\n",
    "        feature = data[:, i].toarray().flatten() \n",
    "\n",
    "        # collects the instances of the feature being present in positive and negative class resp.\n",
    "        pos_feature = feature * labels\n",
    "        neg_feature = feature * inverted_labels\n",
    "        pos_distribution.append((np.mean(pos_feature) + alpha, np.std(pos_feature) + alpha))\n",
    "        neg_distribution.append((np.mean(neg_feature) + alpha, np.std(neg_feature) + alpha))\n",
    "    return pos_distribution, neg_distribution\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Calculates the likelihoods for both classes given the training data as well as priors for both classes\n",
    "\"\"\"\n",
    "def train_gaussian_bayes(X_train, y_train):\n",
    "    # an inverted label array, denoting 1 for the negative class and 0 for the positive\n",
    "    inverted_y_train = np.array([not y for y in y_train]).astype(int)\n",
    "\n",
    "    pos_distribution, neg_distribution = calculate_guassian_distributions(X_train, y_train)\n",
    "    pos_log_distribution = np.log(pos_distribution)\n",
    "    neg_log_distribution = np.log(neg_distribution)\n",
    "\n",
    "    pos_prior = np.sum(y_train) / len(y_train)\n",
    "    neg_prior = np.sum(inverted_y_train) / len(inverted_y_train)\n",
    "    pos_log_prior = np.log(pos_prior)\n",
    "    neg_log_prior = np.log(neg_prior)\n",
    "\n",
    "    return pos_log_distribution, neg_log_distribution, pos_log_prior, neg_log_prior"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fits value (the TF-IDF score for a given feature in the input document) to the guassian distribution of said feature \n",
    "def gaussian(mean, sd, value):\n",
    "    exponent = (- (value - mean)**2 ) / (2 * sd**2)\n",
    "    value = (1 / np.sqrt(2 * np.pi * sd**2)) * np.exp(exponent)\n",
    "    if np.isnan(value): # 0 if NaN\n",
    "        return 0\n",
    "    return value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# produces a class label for a given document using our GNB\n",
    "def get_gaussian_class_label(data, document):\n",
    "    pos_distribution, neg_distribution, pos_log_prior, neg_log_prior = data\n",
    "    features = np.nonzero(document)[0]\n",
    "    pos_total = 0\n",
    "    neg_total = 0\n",
    "    for index in features: # calculates the likelihood using the mean, sd, and value for each feature\n",
    "                           # usage: gaussian(mean, sd, x)\n",
    "                           # pos_distribution[i] = (mean, sd), where i is feature index\n",
    "        pos_total += gaussian(pos_distribution[index][0], pos_distribution[index][1], \n",
    "                            document[index] ) \n",
    "        neg_total += gaussian(neg_distribution[index][0], neg_distribution[index][1],\n",
    "                            document[index] ) \n",
    "    pos_total += pos_log_prior\n",
    "    neg_total += neg_log_prior\n",
    "    label = 1 if pos_total > neg_total else 0\n",
    "    return label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# full GNB pipeline\n",
    "def test_train_gaussian_bayes(train_data, train_labels, test_data):\n",
    "    data = train_gaussian_bayes(train_data, train_labels)\n",
    "    predictions = []\n",
    "    for i, v in enumerate(test_data):\n",
    "        doc =  test_data[i].toarray().flatten()\n",
    "        label = get_gaussian_class_label(data, doc)\n",
    "        predictions.append(label)\n",
    "    return predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluation on development set for Guassian Bayes using stemmed unigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.8645833333333334\n",
      "precision: 0.8291666666666667\n",
      "recall: 0.8923766816143498\n",
      "f1 score: 0.8596112311015119\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions = test_train_gaussian_bayes(X_train, y_train, X_dev)\n",
    "evaluate_model(y_dev, predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sklearn MNB and GNB Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sklearn Multinomial Bayes\n",
      "accuracy: 0.85625\n",
      "precision: 0.8155737704918032\n",
      "recall: 0.8923766816143498\n",
      "f1 score: 0.8522483940042827\n",
      "\n",
      "Sklearn Gaussian Bayes\n",
      "accuracy: 0.65\n",
      "precision: 0.611336032388664\n",
      "recall: 0.6771300448430493\n",
      "f1 score: 0.6425531914893617\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "\n",
    "# full pipeline for training and evaluation of sklearn models\n",
    "def test_train_sklearn_models(X_train, y_train, X_test, y_test):\n",
    "    clf = MultinomialNB()\n",
    "    clf.fit(X_train, y_train)\n",
    "    predictions = clf.predict(X_test)\n",
    "    print(\"Sklearn Multinomial Bayes\")\n",
    "    evaluate_model(y_test, predictions)\n",
    "\n",
    "    clf2 = GaussianNB()\n",
    "    clf2.fit(X_train.toarray(), y_train)\n",
    "    predictions = clf2.predict(X_test.toarray())\n",
    "    print(\"Sklearn Gaussian Bayes\")\n",
    "    evaluate_model(y_test, predictions)\n",
    "test_train_sklearn_models(X_train, y_train, X_dev, y_dev)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For stemmed unigrams our own implementation of multinomial bayes achieves a similar accuracy and f1score, higher precision and a lower recall in comparison to the prebuilt sklearn model. Own implementation of Gaussian Bayes far outperforms sklearn's model on accuracy, precision, and f1 score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNB and GNB Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# full training and evaluation for a given corpus using all models\n",
    "def evaluate_on_corpus(corpus):\n",
    "    shared_vocabulary = generate_shared_vocabulary(corpus)\n",
    "    tf_idf_matrix = generate_tf_idf_matrix(corpus, shared_vocabulary)\n",
    "    X_train, y_train, X_test, y_test, X_dev, y_dev = get_test_train_dev_split(tf_idf_matrix)\n",
    "\n",
    "    multinomial_predictions = test_train_multinomial_bayes(X_train, y_train, X_dev)\n",
    "    print(\"Multinomial Bayes\")\n",
    "    evaluate_model(y_dev, multinomial_predictions)\n",
    "    gaussian_predictions = test_train_gaussian_bayes(X_train, y_train, X_dev)\n",
    "    print(\"Gaussian Bayes\")\n",
    "    evaluate_model(y_dev, gaussian_predictions)\n",
    "    test_train_sklearn_models(X_train, y_train, X_dev, y_dev)\n",
    "    return tf_idf_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In testing our best feature set was unigrams with stemming. Let's now evaluate on the test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Multinomial Bayes\n",
      "accuracy: 0.80875\n",
      "precision: 0.8768328445747801\n",
      "recall: 0.7292682926829268\n",
      "f1 score: 0.796271637816245\n",
      "\n",
      "Gaussian Bayes\n",
      "accuracy: 0.8275\n",
      "precision: 0.8105022831050228\n",
      "recall: 0.8658536585365854\n",
      "f1 score: 0.8372641509433961\n",
      "\n",
      "Sklearn Multinomial Bayes\n",
      "accuracy: 0.82\n",
      "precision: 0.8093023255813954\n",
      "recall: 0.848780487804878\n",
      "f1 score: 0.8285714285714286\n",
      "\n",
      "Sklearn Gaussian Bayes\n",
      "accuracy: 0.6525\n",
      "precision: 0.6578947368421053\n",
      "recall: 0.6707317073170732\n",
      "f1 score: 0.6642512077294686\n",
      "\n"
     ]
    }
   ],
   "source": [
    "X_train, y_train, X_test, y_test, X_dev, y_dev = get_test_train_dev_split(stem_unigram_tf_idf)\n",
    "multinomial_predictions = test_train_multinomial_bayes(X_train, y_train, X_test)\n",
    "print(\"Multinomial Bayes\")\n",
    "evaluate_model(y_test, multinomial_predictions)\n",
    "gaussian_predictions = test_train_gaussian_bayes(X_train, y_train, X_test)\n",
    "print(\"Gaussian Bayes\")\n",
    "evaluate_model(y_test, gaussian_predictions)\n",
    "test_train_sklearn_models(X_train, y_train, X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's first compare the use of one hot matrices to our usual TF-IDF vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1. 1. 1. ... 0. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "one_hot_matrix = generate_tf_idf_matrix(stemmed_unigrams, shared_vocabulary, one_hot=True)\n",
    "print(one_hot_matrix[0].toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8104166666666667"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "# test base logistic regression model's accuracy using tf-idf features\n",
    "X_train, y_train, X_test, y_test, X_dev, y_dev = get_test_train_dev_split(stem_unigram_tf_idf)\n",
    "clf = LogisticRegression(random_state=r_seed, solver=\"sag\")\n",
    "clf.fit(X_train, y_train)\n",
    "clf.score(X_dev, y_dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8333333333333334\n"
     ]
    }
   ],
   "source": [
    "# test base logistic regression model's accuracy using one-hot vectors\n",
    "X_train, y_train, X_test, y_test, X_dev, y_dev = get_test_train_dev_split(one_hot_matrix)\n",
    "clf = LogisticRegression(random_state=r_seed, solver=\"sag\")\n",
    "clf.fit(X_train, y_train)\n",
    "clf.score(X_dev, y_dev)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Clearly one hot matrices lead to higher performance. Let's try evaluating for stemmed unigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.8333333333333334\n",
      "precision: 0.8122270742358079\n",
      "recall: 0.8340807174887892\n",
      "f1 score: 0.8230088495575221\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions = clf.predict(X_dev)\n",
    "evaluate_model(y_dev, predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's evaluate lemmatized unigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.8583333333333333\n",
      "precision: 0.8414096916299559\n",
      "recall: 0.8565022421524664\n",
      "f1 score: 0.8488888888888888\n"
     ]
    }
   ],
   "source": [
    "lem_uni_one_hot = generate_tf_idf_matrix(lemmatized_unigrams, \n",
    "                                         generate_shared_vocabulary(lemmatized_unigrams), one_hot=True)\n",
    "X_train, y_train, X_test, y_test, X_dev, y_dev = get_test_train_dev_split(lem_uni_one_hot)\n",
    "clf = LogisticRegression(random_state=r_seed, solver=\"sag\").fit(X_train, y_train)\n",
    "predictions = clf.predict(X_dev)\n",
    "evaluate_model(y_dev, predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally let's evaluate stemmed unigrams without stopword removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.8375\n",
      "precision: 0.8111587982832618\n",
      "recall: 0.8475336322869955\n",
      "f1 score: 0.8289473684210525\n"
     ]
    }
   ],
   "source": [
    "stemmed_unigrams_stopwords = apply_to_corpus(apply_stemming, corps_unigrams_with_stopwords)\n",
    "stem_uni_stop_one_hot = generate_tf_idf_matrix(stemmed_unigrams_stopwords, \n",
    "                                               generate_shared_vocabulary(stemmed_unigrams_stopwords), one_hot=True)\n",
    "X_train, y_train, X_test, y_test, X_dev, y_dev = get_test_train_dev_split(stem_uni_stop_one_hot)\n",
    "clf = LogisticRegression(random_state=r_seed, solver=\"sag\")\n",
    "clf.fit(X_train, y_train)\n",
    "predictions = clf.predict(X_dev)\n",
    "evaluate_model(y_dev, predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The best performing feature set was thus lemmatization with stop-word removal"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Support Vector Machines"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's define our SVM classifier and evaluate stemmed unigrams with stopword removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.8520833333333333\n",
      "precision: 0.8247863247863247\n",
      "recall: 0.8654708520179372\n",
      "f1 score: 0.8446389496717724\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn import svm\n",
    "\n",
    "# svm classifer train and evaluation pipeline\n",
    "def svm_classifier(feature_matrix):\n",
    "    X_train, y_train, X_test, y_test, X_dev, y_dev = get_test_train_dev_split(feature_matrix)\n",
    "    clf = svm.SVC()\n",
    "    clf.fit(X_train, y_train)\n",
    "    predictions = clf.predict(X_dev)\n",
    "    evaluate_model(y_dev, predictions)\n",
    "    \n",
    "svm_classifier(one_hot_matrix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SVM with lemmatized unigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.85\n",
      "precision: 0.8212765957446808\n",
      "recall: 0.8654708520179372\n",
      "f1 score: 0.8427947598253275\n",
      "\n"
     ]
    }
   ],
   "source": [
    "svm_classifier(lem_uni_one_hot)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SVM with stemmed unigrams and no stopword removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.8520833333333333\n",
      "precision: 0.8247863247863247\n",
      "recall: 0.8654708520179372\n",
      "f1 score: 0.8446389496717724\n",
      "\n"
     ]
    }
   ],
   "source": [
    "svm_classifier(stem_uni_stop_one_hot)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Our highest performing set for the SVM classifier was lemmatization with stopword removal"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Logistic Regression Hyperparameter Optimisation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For our LogReg classifier we found that parallelizing does not increase performance. The tuned hyperparameters of our fine tuned model were as follows\n",
    "+ C = 1.5\n",
    "+ solver = \"lbfgs\"\n",
    "+ penalty = \"l2\"\n",
    "+ n_jobs = None (i.e. not parallelized)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.83625\n",
      "precision: 0.8329355608591885\n",
      "recall: 0.8512195121951219\n",
      "f1 score: 0.8419782870928829\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(C=1.5, \n",
    "                         solver=\"lbfgs\", \n",
    "                         random_state=r_seed, \n",
    "                         penalty=\"l2\", \n",
    "                         n_jobs=None)\n",
    "clf.fit(X_train, y_train)\n",
    "predictions = clf.predict(X_test)\n",
    "evaluate_model(y_test, predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SVM Hyperparameter Optimisation </h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The final tuned hyperparameters of our SVM model were as follows\n",
    "+ C = 0.9\n",
    "+ kernel = \"rbf\" \n",
    "+ gamma = \"scale\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.82625\n",
      "precision: 0.8086560364464692\n",
      "recall: 0.8658536585365854\n",
      "f1 score: 0.8362779740871613\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clf = svm.SVC(C=0.9, \n",
    "              kernel=\"rbf\", \n",
    "              gamma=\"scale\")\n",
    "clf.fit(X_train, y_train)\n",
    "predictions = clf.predict(X_test)\n",
    "evaluate_model(y_test, predictions)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}