03_earthquake_imputation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.experimental import enable_iterative_imputer\n",
    "from sklearn.impute import IterativeImputer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "quake_frame = pd.read_csv('data/consolidated_data.csv')\n",
    "\n",
    "quake_frame['simple_label'] = quake_frame['type'] != 'earthquake'\n",
    "\n",
    "quake_frame.drop(['id', 'Unnamed: 0', 'place', 'time', 'updated', 'type'], inplace=True, axis=1)\n",
    "\n",
    "quake_frame['has_null'] = quake_frame.isna().sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     1227408\n",
       "6      578546\n",
       "1      466870\n",
       "3      278200\n",
       "5      174823\n",
       "4      148396\n",
       "2      145856\n",
       "7      115921\n",
       "8       79304\n",
       "9       41584\n",
       "10      15865\n",
       "11          1\n",
       "Name: has_null, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quake_frame['has_null'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Iterative imputation\n",
    "\n",
    "Our Random Forest is still for two classes that takes only rows that have no nans in them. This time, we'll use the iterative imputer. Let's see how many we get.  \n",
    "Then we'll split the data 80/20 and run training.  \n",
    "Problem is, we'll be cheating if we impute on all the data and then test on imputed data - if we get the imputation wrong, we could not notice how far away our model is from actual performance until it hits real world data. Thus the validation set needs to be imputed data only. We'll do this by imputing on all and then take validation data only from a part of the dataset that had no nulls to begin with.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>latitude</th>\n",
       "      <td>3272774.0</td>\n",
       "      <td>35.720738</td>\n",
       "      <td>20.256723</td>\n",
       "      <td>-84.422</td>\n",
       "      <td>34.11800</td>\n",
       "      <td>37.576167</td>\n",
       "      <td>42.258667</td>\n",
       "      <td>87.265</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>longitude</th>\n",
       "      <td>3272774.0</td>\n",
       "      <td>-92.856671</td>\n",
       "      <td>80.553255</td>\n",
       "      <td>-179.999</td>\n",
       "      <td>-122.79583</td>\n",
       "      <td>-118.811167</td>\n",
       "      <td>-115.454167</td>\n",
       "      <td>180.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>depth</th>\n",
       "      <td>3272765.0</td>\n",
       "      <td>22.334946</td>\n",
       "      <td>56.320328</td>\n",
       "      <td>-10.000</td>\n",
       "      <td>3.00200</td>\n",
       "      <td>7.155000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>735.800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mag</th>\n",
       "      <td>3116325.0</td>\n",
       "      <td>1.878941</td>\n",
       "      <td>1.352506</td>\n",
       "      <td>-9.990</td>\n",
       "      <td>0.97000</td>\n",
       "      <td>1.500000</td>\n",
       "      <td>2.460000</td>\n",
       "      <td>9.100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nst</th>\n",
       "      <td>2391208.0</td>\n",
       "      <td>15.601496</td>\n",
       "      <td>26.606866</td>\n",
       "      <td>0.000</td>\n",
       "      <td>5.00000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>934.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gap</th>\n",
       "      <td>2434225.0</td>\n",
       "      <td>130.487608</td>\n",
       "      <td>69.710621</td>\n",
       "      <td>0.000</td>\n",
       "      <td>79.00000</td>\n",
       "      <td>115.000000</td>\n",
       "      <td>168.260000</td>\n",
       "      <td>360.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dmin</th>\n",
       "      <td>1926032.0</td>\n",
       "      <td>0.255999</td>\n",
       "      <td>1.333459</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.02093</td>\n",
       "      <td>0.051350</td>\n",
       "      <td>0.116000</td>\n",
       "      <td>141.160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rms</th>\n",
       "      <td>3061121.0</td>\n",
       "      <td>0.315205</td>\n",
       "      <td>0.399901</td>\n",
       "      <td>-1.000</td>\n",
       "      <td>0.06000</td>\n",
       "      <td>0.150000</td>\n",
       "      <td>0.480000</td>\n",
       "      <td>104.330</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>horizontalError</th>\n",
       "      <td>1740811.0</td>\n",
       "      <td>1.266841</td>\n",
       "      <td>3.168282</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.30000</td>\n",
       "      <td>0.480000</td>\n",
       "      <td>0.930000</td>\n",
       "      <td>280.600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>depthError</th>\n",
       "      <td>2666089.0</td>\n",
       "      <td>5.640320</td>\n",
       "      <td>1167.801181</td>\n",
       "      <td>-1.000</td>\n",
       "      <td>0.49000</td>\n",
       "      <td>0.960000</td>\n",
       "      <td>2.760000</td>\n",
       "      <td>1773552.500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>magError</th>\n",
       "      <td>1491762.0</td>\n",
       "      <td>0.166893</td>\n",
       "      <td>0.147429</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.09000</td>\n",
       "      <td>0.142000</td>\n",
       "      <td>0.210000</td>\n",
       "      <td>6.110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>magNst</th>\n",
       "      <td>2283857.0</td>\n",
       "      <td>12.607386</td>\n",
       "      <td>21.127239</td>\n",
       "      <td>0.000</td>\n",
       "      <td>3.00000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>941.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>has_null</th>\n",
       "      <td>3272774.0</td>\n",
       "      <td>2.600532</td>\n",
       "      <td>2.747433</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>11.000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     count        mean          std      min        25%  \\\n",
       "latitude         3272774.0   35.720738    20.256723  -84.422   34.11800   \n",
       "longitude        3272774.0  -92.856671    80.553255 -179.999 -122.79583   \n",
       "depth            3272765.0   22.334946    56.320328  -10.000    3.00200   \n",
       "mag              3116325.0    1.878941     1.352506   -9.990    0.97000   \n",
       "nst              2391208.0   15.601496    26.606866    0.000    5.00000   \n",
       "gap              2434225.0  130.487608    69.710621    0.000   79.00000   \n",
       "dmin             1926032.0    0.255999     1.333459    0.000    0.02093   \n",
       "rms              3061121.0    0.315205     0.399901   -1.000    0.06000   \n",
       "horizontalError  1740811.0    1.266841     3.168282    0.000    0.30000   \n",
       "depthError       2666089.0    5.640320  1167.801181   -1.000    0.49000   \n",
       "magError         1491762.0    0.166893     0.147429    0.000    0.09000   \n",
       "magNst           2283857.0   12.607386    21.127239    0.000    3.00000   \n",
       "has_null         3272774.0    2.600532     2.747433    0.000    0.00000   \n",
       "\n",
       "                        50%         75%          max  \n",
       "latitude          37.576167   42.258667       87.265  \n",
       "longitude       -118.811167 -115.454167      180.000  \n",
       "depth              7.155000   15.000000      735.800  \n",
       "mag                1.500000    2.460000        9.100  \n",
       "nst               10.000000   18.000000      934.000  \n",
       "gap              115.000000  168.260000      360.000  \n",
       "dmin               0.051350    0.116000      141.160  \n",
       "rms                0.150000    0.480000      104.330  \n",
       "horizontalError    0.480000    0.930000      280.600  \n",
       "depthError         0.960000    2.760000  1773552.500  \n",
       "magError           0.142000    0.210000        6.110  \n",
       "magNst             7.000000   14.000000      941.000  \n",
       "has_null           1.000000    6.000000       11.000  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quake_frame.describe().transpose()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We better limit that depthError value. It will still be an outlier, but shouldn't make things too crazy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "quake_frame['depthError'].clip(upper=10000, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In order to be able to encode all the categoricals, we'll have to transform only the values that are not missing. I found a label encoder class on StackOverflow that should help sorting this out.  \n",
    "We'll run that encoding and then replace the strings with pandas NaNs so the column can be considered integer and we can impute on it.  \n",
    "We're not scaling and one-hot encoding because that would drive the number of columns beyond 600 and I do not have the compute to deal with that. It would be interesting to see a result, though!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator\n",
    "from sklearn.base import TransformerMixin\n",
    "\n",
    "class LabelEncoderByCol(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self,col):\n",
    "        #List of column names in the DataFrame that should be encoded\n",
    "        self.col = col\n",
    "        #Dictionary storing a LabelEncoder for each column\n",
    "        self.le_dic = {}\n",
    "        for el in self.col:\n",
    "            self.le_dic[el] = LabelEncoder()\n",
    "\n",
    "    def fit(self,x,y=None):\n",
    "        #Fill missing values with the string 'NaN'\n",
    "        x[self.col] = x[self.col].fillna('NaN')\n",
    "        for el in self.col:\n",
    "            #Only use the values that are not 'NaN' to fit the Encoder\n",
    "            a = x[el][x[el]!='NaN']\n",
    "            self.le_dic[el].fit(a)\n",
    "        return self\n",
    "\n",
    "    def transform(self,x,y=None):\n",
    "        #Fill missing values with the string 'NaN'\n",
    "        x[self.col] = x[self.col].fillna('NaN')\n",
    "        for el in self.col:\n",
    "            #Only use the values that are not 'NaN' to fit the Encoder\n",
    "            a = x[el][x[el]!='NaN']\n",
    "            #Store an ndarray of the current column\n",
    "            b = x[el].values\n",
    "            #Replace the elements in the ndarray that are not 'NaN'\n",
    "            #using the transformer\n",
    "            b[b!='NaN'] = self.le_dic[el].transform(a)\n",
    "            #Overwrite the column in the DataFrame\n",
    "            x[el]=b\n",
    "        #return the transformed DataFrame\n",
    "        return x\n",
    "    \n",
    "def create_label_encoder_by_column(data, cols=None):\n",
    "    if not cols:\n",
    "        cols = data.columns.to_list()\n",
    "    ### fit with the desired col, col in position 0 for this example\n",
    "    fit_list = [data[col].unique() for col in cols]\n",
    "    enc_list = [LabelEncoder().fit(col) for col in fit_list]\n",
    "    return enc_list\n",
    "\n",
    "def transform_all_but_nans(data, cols=None, name='_enc'):\n",
    "    if not cols:\n",
    "        cols = data.columns.to_list()\n",
    "    encs = create_label_encoder_by_column(data, cols)\n",
    "    for col, enc in zip(cols, encs):\n",
    "        data[col + name] = data[col].apply(lambda x: enc.transform(x) if x is not None else x)\n",
    "    return data\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>depth</th>\n",
       "      <th>mag</th>\n",
       "      <th>magType</th>\n",
       "      <th>nst</th>\n",
       "      <th>gap</th>\n",
       "      <th>dmin</th>\n",
       "      <th>rms</th>\n",
       "      <th>net</th>\n",
       "      <th>horizontalError</th>\n",
       "      <th>depthError</th>\n",
       "      <th>magError</th>\n",
       "      <th>magNst</th>\n",
       "      <th>status</th>\n",
       "      <th>locationSource</th>\n",
       "      <th>magSource</th>\n",
       "      <th>simple_label</th>\n",
       "      <th>has_null</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>37.003502</td>\n",
       "      <td>-117.996834</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>17</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>31</td>\n",
       "      <td>54</td>\n",
       "      <td>True</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>35.642788</td>\n",
       "      <td>-120.933601</td>\n",
       "      <td>5.00</td>\n",
       "      <td>1.99</td>\n",
       "      <td>17</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>31</td>\n",
       "      <td>54</td>\n",
       "      <td>False</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>34.164520</td>\n",
       "      <td>-118.185036</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>17</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>31</td>\n",
       "      <td>54</td>\n",
       "      <td>False</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>33.836494</td>\n",
       "      <td>-116.781868</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>17</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>31</td>\n",
       "      <td>54</td>\n",
       "      <td>True</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>33.208477</td>\n",
       "      <td>-115.476997</td>\n",
       "      <td>5.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>17</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>31</td>\n",
       "      <td>54</td>\n",
       "      <td>True</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3272769</th>\n",
       "      <td>61.417200</td>\n",
       "      <td>-147.564900</td>\n",
       "      <td>13.10</td>\n",
       "      <td>1.20</td>\n",
       "      <td>18</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.40</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3272770</th>\n",
       "      <td>66.227700</td>\n",
       "      <td>-157.202600</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.80</td>\n",
       "      <td>18</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.01</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.40</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3272771</th>\n",
       "      <td>33.234667</td>\n",
       "      <td>-116.771167</td>\n",
       "      <td>12.45</td>\n",
       "      <td>0.65</td>\n",
       "      <td>18</td>\n",
       "      <td>19.0</td>\n",
       "      <td>54.0</td>\n",
       "      <td>0.01048</td>\n",
       "      <td>0.16</td>\n",
       "      <td>3</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.158</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0</td>\n",
       "      <td>31</td>\n",
       "      <td>54</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3272772</th>\n",
       "      <td>62.829900</td>\n",
       "      <td>-148.766400</td>\n",
       "      <td>55.50</td>\n",
       "      <td>1.80</td>\n",
       "      <td>18</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.51</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.60</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3272773</th>\n",
       "      <td>33.036333</td>\n",
       "      <td>-116.436000</td>\n",
       "      <td>5.48</td>\n",
       "      <td>1.19</td>\n",
       "      <td>18</td>\n",
       "      <td>39.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>0.14280</td>\n",
       "      <td>0.23</td>\n",
       "      <td>3</td>\n",
       "      <td>0.33</td>\n",
       "      <td>1.64</td>\n",
       "      <td>0.142</td>\n",
       "      <td>28.0</td>\n",
       "      <td>0</td>\n",
       "      <td>31</td>\n",
       "      <td>54</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3272774 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          latitude   longitude  depth   mag magType   nst   gap     dmin  \\\n",
       "0        37.003502 -117.996834   0.00  0.00      17   0.0   NaN      NaN   \n",
       "1        35.642788 -120.933601   5.00  1.99      17   2.0   NaN      NaN   \n",
       "2        34.164520 -118.185036   0.00  0.00      17   NaN   NaN      NaN   \n",
       "3        33.836494 -116.781868   0.00  0.00      17   NaN   NaN      NaN   \n",
       "4        33.208477 -115.476997   5.00  0.00      17   NaN   NaN      NaN   \n",
       "...            ...         ...    ...   ...     ...   ...   ...      ...   \n",
       "3272769  61.417200 -147.564900  13.10  1.20      18   NaN   NaN      NaN   \n",
       "3272770  66.227700 -157.202600   0.00  1.80      18   NaN   NaN      NaN   \n",
       "3272771  33.234667 -116.771167  12.45  0.65      18  19.0  54.0  0.01048   \n",
       "3272772  62.829900 -148.766400  55.50  1.80      18   NaN   NaN      NaN   \n",
       "3272773  33.036333 -116.436000   5.48  1.19      18  39.0  66.0  0.14280   \n",
       "\n",
       "          rms  net  horizontalError  depthError  magError  magNst status  \\\n",
       "0         NaN    3              NaN         NaN       NaN     0.0      2   \n",
       "1         NaN    3              NaN         NaN       NaN     0.0      2   \n",
       "2         NaN    3              NaN         NaN       NaN     0.0      2   \n",
       "3         NaN    3              NaN         NaN       NaN     0.0      2   \n",
       "4         NaN    3              NaN         NaN       NaN     0.0      2   \n",
       "...       ...  ...              ...         ...       ...     ...    ...   \n",
       "3272769  0.66    0              NaN        0.40       NaN     NaN      0   \n",
       "3272770  1.01    0              NaN        0.40       NaN     NaN      0   \n",
       "3272771  0.16    3             0.27        0.70     0.158    14.0      0   \n",
       "3272772  0.51    0              NaN        1.60       NaN     NaN      0   \n",
       "3272773  0.23    3             0.33        1.64     0.142    28.0      0   \n",
       "\n",
       "         locationSource  magSource  simple_label  has_null  \n",
       "0                    31         54          True         6  \n",
       "1                    31         54         False         6  \n",
       "2                    31         54         False         7  \n",
       "3                    31         54          True         7  \n",
       "4                    31         54          True         7  \n",
       "...                 ...        ...           ...       ...  \n",
       "3272769               5         10         False         6  \n",
       "3272770               5         10         False         6  \n",
       "3272771              31         54         False         0  \n",
       "3272772               5         10         False         6  \n",
       "3272773              31         54         False         0  \n",
       "\n",
       "[3272774 rows x 19 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "col = ['magType', 'net', 'status', 'locationSource', 'magSource']\n",
    "le = LabelEncoderByCol(col=col)\n",
    "\n",
    "le.fit(quake_frame)\n",
    "\n",
    "le.transform(quake_frame)\n",
    "\n",
    "quake_frame.replace('NaN', pd.NA)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll grab all rows that have fewer than 5 nulls to make life easier for our imputer (and prevent it from having to just guess)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "quake_frame_imputation = quake_frame.loc[quake_frame['has_null'] < 5,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/eikegermann/anaconda3/envs/earthquake/lib/python3.8/site-packages/pandas/core/indexing.py:966: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.obj[item] = s\n"
     ]
    }
   ],
   "source": [
    "imp = IterativeImputer(max_iter=20, random_state=42)\n",
    "imp_cols = quake_frame.columns.to_list()\n",
    "\n",
    "quake_frame_imputation.loc[:, imp_cols] = imp.fit_transform(quake_frame_imputation.loc[:, imp_cols])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>depth</th>\n",
       "      <th>mag</th>\n",
       "      <th>magType</th>\n",
       "      <th>nst</th>\n",
       "      <th>gap</th>\n",
       "      <th>dmin</th>\n",
       "      <th>rms</th>\n",
       "      <th>net</th>\n",
       "      <th>horizontalError</th>\n",
       "      <th>depthError</th>\n",
       "      <th>magError</th>\n",
       "      <th>magNst</th>\n",
       "      <th>status</th>\n",
       "      <th>locationSource</th>\n",
       "      <th>magSource</th>\n",
       "      <th>simple_label</th>\n",
       "      <th>has_null</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>46.276500</td>\n",
       "      <td>-118.360000</td>\n",
       "      <td>-0.260</td>\n",
       "      <td>2.30</td>\n",
       "      <td>14.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>303.0</td>\n",
       "      <td>0.520000</td>\n",
       "      <td>0.14</td>\n",
       "      <td>19.0</td>\n",
       "      <td>0.410</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.120</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>190.0</td>\n",
       "      <td>319.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>46.332833</td>\n",
       "      <td>-118.391167</td>\n",
       "      <td>-0.260</td>\n",
       "      <td>2.60</td>\n",
       "      <td>14.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>299.0</td>\n",
       "      <td>0.481400</td>\n",
       "      <td>0.15</td>\n",
       "      <td>19.0</td>\n",
       "      <td>0.144</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.210</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>190.0</td>\n",
       "      <td>319.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>32.707167</td>\n",
       "      <td>-115.417000</td>\n",
       "      <td>6.000</td>\n",
       "      <td>2.75</td>\n",
       "      <td>17.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>214.0</td>\n",
       "      <td>0.603600</td>\n",
       "      <td>0.58</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.600</td>\n",
       "      <td>31.61</td>\n",
       "      <td>0.084</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>54.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>37.433333</td>\n",
       "      <td>-118.743500</td>\n",
       "      <td>6.000</td>\n",
       "      <td>3.69</td>\n",
       "      <td>18.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>211.0</td>\n",
       "      <td>0.549249</td>\n",
       "      <td>0.58</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.240</td>\n",
       "      <td>31.61</td>\n",
       "      <td>0.224</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>54.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>46.749500</td>\n",
       "      <td>-119.371500</td>\n",
       "      <td>1.869</td>\n",
       "      <td>1.50</td>\n",
       "      <td>14.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>93.0</td>\n",
       "      <td>0.106300</td>\n",
       "      <td>0.15</td>\n",
       "      <td>19.0</td>\n",
       "      <td>0.578</td>\n",
       "      <td>1.91</td>\n",
       "      <td>0.120</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>190.0</td>\n",
       "      <td>319.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     latitude   longitude  depth   mag  magType   nst    gap      dmin   rms  \\\n",
       "7   46.276500 -118.360000 -0.260  2.30     14.0   6.0  303.0  0.520000  0.14   \n",
       "8   46.332833 -118.391167 -0.260  2.60     14.0   6.0  299.0  0.481400  0.15   \n",
       "9   32.707167 -115.417000  6.000  2.75     17.0   4.0  214.0  0.603600  0.58   \n",
       "11  37.433333 -118.743500  6.000  3.69     18.0  10.0  211.0  0.549249  0.58   \n",
       "12  46.749500 -119.371500  1.869  1.50     14.0   6.0   93.0  0.106300  0.15   \n",
       "\n",
       "     net  horizontalError  depthError  magError  magNst  status  \\\n",
       "7   19.0            0.410        0.24     0.120     0.0     2.0   \n",
       "8   19.0            0.144        0.09     0.210     0.0     2.0   \n",
       "9    3.0            4.600       31.61     0.084     6.0     2.0   \n",
       "11   3.0            3.240       31.61     0.224     7.0     2.0   \n",
       "12  19.0            0.578        1.91     0.120     0.0     2.0   \n",
       "\n",
       "    locationSource  magSource  simple_label  has_null  \n",
       "7            190.0      319.0           0.0       0.0  \n",
       "8            190.0      319.0           0.0       0.0  \n",
       "9             31.0       54.0           0.0       0.0  \n",
       "11            31.0       54.0           0.0       1.0  \n",
       "12           190.0      319.0           0.0       0.0  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quake_frame_imputation.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/eikegermann/anaconda3/envs/earthquake/lib/python3.8/site-packages/pandas/core/indexing.py:671: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_with_indexer(indexer, value)\n",
      "<ipython-input-11-97de2e6df0ae>:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  quake_frame_imputation.loc[:, col] = quake_frame_imputation[col].round(decimals=0)\n",
      "<ipython-input-11-97de2e6df0ae>:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  quake_frame_imputation.loc[:, col] = quake_frame_imputation[col].astype(int)\n"
     ]
    }
   ],
   "source": [
    "quake_frame_imputation.loc[:, col] = quake_frame_imputation[col].round(decimals=0)\n",
    "quake_frame_imputation.loc[:, col] = quake_frame_imputation[col].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 14.,  17.,  18.,  24.,   8.,  13.,  15.,  23.,  16.,   1.,  12.,\n",
       "        10.,  11.,  -2.,   7.,   5.,  -4.,   6.,   4.,   9.,  21.,  19.,\n",
       "         2., -23.,  26.,  25.,  27.,  29.,  28.,   3.,   0.,  22.,  20.])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quake_frame_imputation.magType.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.03235541948092627"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(quake_frame_imputation.simple_label)/len(quake_frame_imputation.index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Okay, now let's see how this works out. We'll grab the first 20% of the dataframe when order by has_nulls to get original data (this should be good enough for our purposes as a comparison, properly, it would have to be randomised and then selected to avoid any effects from the ordering)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "quake_frame_imputation = quake_frame_imputation.sample(frac=1, random_state=42).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "quake_frame_imputation.sort_values(by=['has_null'], inplace=True, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['latitude', 'longitude', 'depth', 'mag', 'magType', 'nst', 'gap',\n",
       "       'dmin', 'rms', 'net', 'horizontalError', 'depthError', 'magError',\n",
       "       'magNst', 'status', 'locationSource', 'magSource', 'simple_label',\n",
       "       'has_null'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quake_frame_imputation.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_cols = ['latitude',\n",
    " 'longitude',\n",
    " 'depth',\n",
    " 'mag',\n",
    " 'nst',\n",
    " 'gap',\n",
    " 'dmin',\n",
    " 'rms',\n",
    " 'horizontalError',\n",
    " 'depthError',\n",
    " 'magError',\n",
    " 'magNst',\n",
    " 'magType',\n",
    " 'net',\n",
    " 'status',\n",
    " 'locationSource',\n",
    " 'magSource']\n",
    "\n",
    "y_col = ['simple_label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Separate train and valid sets and shuffle training set\n",
    "valid_length = int(np.round(len(quake_frame_imputation.index) * 0.2))\n",
    "quake_frame_imp_valid = quake_frame_imputation.loc[:valid_length, :]\n",
    "quake_frame_imp_train = quake_frame_imputation.loc[valid_length:, :]\n",
    "quake_frame_imp_train = quake_frame_imp_train.sample(frac=1, random_state=42).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_X = quake_frame_imp_valid.loc[:, x_cols]\n",
    "valid_y = quake_frame_imp_valid.loc[:, y_col]\n",
    "\n",
    "train_X = quake_frame_imp_train.loc[:, x_cols]\n",
    "train_y = quake_frame_imp_train.loc[:, y_col]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_estim = 100\n",
    "\n",
    "rfc = RandomForestClassifier(n_estimators=n_estim,\n",
    "                             random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,\n",
       "                       criterion='gini', max_depth=None, max_features='auto',\n",
       "                       max_leaf_nodes=None, max_samples=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=2,\n",
       "                       min_weight_fraction_leaf=0.0, n_estimators=100,\n",
       "                       n_jobs=None, oob_score=False, random_state=42, verbose=0,\n",
       "                       warm_start=False)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rfc.fit(train_X, np.ravel(train_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import (\n",
    "    precision_score, recall_score, roc_auc_score, f1_score, confusion_matrix, accuracy_score\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = pd.DataFrame(rfc.predict(valid_X), columns=['predictions'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision:  0.9648824887922837\n",
      "Recall:  0.8823529411764706\n",
      "ROC score:  0.9405852729155186\n",
      "F1 score:  0.9217741150514259\n",
      "Accuracy score:  0.994681777975811\n"
     ]
    }
   ],
   "source": [
    "prec = precision_score(valid_y, preds)\n",
    "reca = recall_score(valid_y, preds)\n",
    "roc = roc_auc_score(valid_y, preds)\n",
    "f1 = f1_score(valid_y, preds)\n",
    "acc = accuracy_score(valid_y, preds)\n",
    "conf_mat = confusion_matrix(valid_y, preds)\n",
    "\n",
    "print(\"Precision: \", prec)\n",
    "print(\"Recall: \", reca)\n",
    "print(\"ROC score: \", roc)\n",
    "print(\"F1 score: \", f1)\n",
    "print(\"Accuracy score: \", acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}