exploratory_data_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "import gensim\n",
    "from gensim.models import LdaModel\n",
    "from gensim.corpora import Dictionary\n",
    "import pyLDAvis.gensim\n",
    "\n",
    "from snippets import (getTopWordsByCategory, \n",
    "                      plotTopWordsByCategory, plotTopWords, \n",
    "                      plotClassShares, printSampleComments,\n",
    "                      plotSetIntersections, calculateUncertanityCoeff,\n",
    "                      plotUncertanityCoeff)\n",
    "\n",
    "from features_engineering import (calculateTFIDFscore, split_words)\n",
    "from nb_svm import fitModel, NbSvmClassifier\n",
    "from load_data import loadData\n",
    "\n",
    "from config import DATA_FILE, LABELS, CONTENT, UNIQUE_ID, TEST_SIZE, C, NUM_TOPICS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = loadData(DATA_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[CONTENT].fillna(\"unknown\", inplace=True)\n",
    "\n",
    "for col in LABELS:\n",
    "    print(\"Labels for %s are: %s\" % (col, ', '.join(map(str, df[col].unique()))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "INNOCENT_LABEL = \"innocent\"\n",
    "df[INNOCENT_LABEL] = (df[LABELS].sum(axis = 1) == 0).astype(int)\n",
    "ALL_LABELS = LABELS + [INNOCENT_LABEL]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sample comments\n",
    "printSampleComments(df, CONTENT, LABELS, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Analyse share of each label\n",
    "ratio = df[LABELS].mean()\n",
    "plotClassShares(LABELS, ratio)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Multiple labels\n",
    "print(\"There are %s comments, but %s comment labels.\" % (df.shape[0], int(df.shape[0]*ratio.sum())))\n",
    "\n",
    "df_subset = df.loc[df[INNOCENT_LABEL] == 0]    \n",
    "plotSetIntersections(df_subset, LABELS, UNIQUE_ID)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Theil's U uncertanity coefficient \n",
    "# https://towardsdatascience.com/the-search-for-categorical-correlation-a1cf7f1888c9\n",
    "\n",
    "uncertanity_coeff = calculateUncertanityCoeff(df, LABELS)\n",
    "plotUncertanityCoeff(uncertanity_coeff, LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculate TF-IDF (Term Frequency - Inverse Document Frequency) \n",
    "word_counts, features = calculateTFIDFscore(df[CONTENT])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get most common words\n",
    "NUM_WORDS_CAT = 10\n",
    "NUM_WORDS_ALL = 20\n",
    "words, counts = getTopWordsByCategory(df, LABELS, word_counts, features, NUM_WORDS_CAT)\n",
    "words_all, counts_all = getTopWordsByCategory(df, LABELS, word_counts, features, NUM_WORDS_ALL, aggregate = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plotTopWords(words_all[0], counts_all[0], \"All\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plotTopWordsByCategory(words, counts, LABELS)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}