04_miss_forest_imputation_pre_processing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loan Prediction 04 - Data Imputation With Random Forest"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let us try to improve the previous results by imputing missing data with a Random Forest.\n",
    "\n",
    "But first, we will remove rows with more than one missing value."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "sys.path.append('utils')\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import math\n",
    "plt.style.use('seaborn')\n",
    "\n",
    "from missingpy import MissForest\n",
    "from sklearn.preprocessing import OrdinalEncoder\n",
    "\n",
    "import dataframe_utils\n",
    "import preprocess_utils"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading original dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(614, 12)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_import = pd.read_csv('dataset/train_loan.csv')\n",
    "df_import.drop(columns=['Loan_ID'],inplace = True)\n",
    "df_import.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Counting missing values by column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Gender               13\n",
       "Married               3\n",
       "Dependents           15\n",
       "Education             0\n",
       "Self_Employed        32\n",
       "ApplicantIncome       0\n",
       "CoapplicantIncome     0\n",
       "LoanAmount           22\n",
       "Loan_Amount_Term     14\n",
       "Credit_History       50\n",
       "Property_Area         0\n",
       "Loan_Status           0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_import.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Rows with one or no missing value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gender</th>\n",
       "      <th>Married</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>Education</th>\n",
       "      <th>Self_Employed</th>\n",
       "      <th>ApplicantIncome</th>\n",
       "      <th>CoapplicantIncome</th>\n",
       "      <th>LoanAmount</th>\n",
       "      <th>Loan_Amount_Term</th>\n",
       "      <th>Credit_History</th>\n",
       "      <th>Property_Area</th>\n",
       "      <th>Loan_Status</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Male</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>5849</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Urban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>1</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>4583</td>\n",
       "      <td>1508.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Rural</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>Yes</td>\n",
       "      <td>3000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Urban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0</td>\n",
       "      <td>Not Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>2583</td>\n",
       "      <td>2358.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Urban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Male</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>6000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Urban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>609</th>\n",
       "      <td>Female</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>2900</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Rural</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>610</th>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>3+</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>4106</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>180.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Rural</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>611</th>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>1</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>8072</td>\n",
       "      <td>240.0</td>\n",
       "      <td>253.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Urban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>612</th>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>2</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>7583</td>\n",
       "      <td>0.0</td>\n",
       "      <td>187.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Urban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>613</th>\n",
       "      <td>Female</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>Yes</td>\n",
       "      <td>4583</td>\n",
       "      <td>0.0</td>\n",
       "      <td>133.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Semiurban</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>601 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Gender Married Dependents     Education Self_Employed  ApplicantIncome  \\\n",
       "0      Male      No          0      Graduate            No             5849   \n",
       "1      Male     Yes          1      Graduate            No             4583   \n",
       "2      Male     Yes          0      Graduate           Yes             3000   \n",
       "3      Male     Yes          0  Not Graduate            No             2583   \n",
       "4      Male      No          0      Graduate            No             6000   \n",
       "..      ...     ...        ...           ...           ...              ...   \n",
       "609  Female      No          0      Graduate            No             2900   \n",
       "610    Male     Yes         3+      Graduate            No             4106   \n",
       "611    Male     Yes          1      Graduate            No             8072   \n",
       "612    Male     Yes          2      Graduate            No             7583   \n",
       "613  Female      No          0      Graduate           Yes             4583   \n",
       "\n",
       "     CoapplicantIncome  LoanAmount  Loan_Amount_Term  Credit_History  \\\n",
       "0                  0.0         NaN             360.0             1.0   \n",
       "1               1508.0       128.0             360.0             1.0   \n",
       "2                  0.0        66.0             360.0             1.0   \n",
       "3               2358.0       120.0             360.0             1.0   \n",
       "4                  0.0       141.0             360.0             1.0   \n",
       "..                 ...         ...               ...             ...   \n",
       "609                0.0        71.0             360.0             1.0   \n",
       "610                0.0        40.0             180.0             1.0   \n",
       "611              240.0       253.0             360.0             1.0   \n",
       "612                0.0       187.0             360.0             1.0   \n",
       "613                0.0       133.0             360.0             0.0   \n",
       "\n",
       "    Property_Area Loan_Status  \n",
       "0           Urban           Y  \n",
       "1           Rural           N  \n",
       "2           Urban           Y  \n",
       "3           Urban           Y  \n",
       "4           Urban           Y  \n",
       "..            ...         ...  \n",
       "609         Rural           Y  \n",
       "610         Rural           Y  \n",
       "611         Urban           Y  \n",
       "612         Urban           Y  \n",
       "613     Semiurban           N  \n",
       "\n",
       "[601 rows x 12 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "null_rows = df_import.isnull().sum(axis = 1)\n",
    "df_rows = df_import.loc[(null_rows == 1) | (null_rows == 0),:]\n",
    "df_rows"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Replacing missing values with MissForest"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let us prepare the dataset in order to execute the MissForest algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/lucasthim/projects/loan-prediction/loan_libs/lib/python3.6/site-packages/pandas/core/ops/__init__.py:1115: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  result = method(y)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gender</th>\n",
       "      <th>Married</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>Education</th>\n",
       "      <th>Self_Employed</th>\n",
       "      <th>ApplicantIncome</th>\n",
       "      <th>CoapplicantIncome</th>\n",
       "      <th>LoanAmount</th>\n",
       "      <th>Loan_Amount_Term</th>\n",
       "      <th>Credit_History</th>\n",
       "      <th>Property_Area</th>\n",
       "      <th>Loan_Status</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5849</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4583</td>\n",
       "      <td>1508.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2583</td>\n",
       "      <td>2358.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>609</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2900</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>610</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4106</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>180.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>611</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8072</td>\n",
       "      <td>240.0</td>\n",
       "      <td>253.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>612</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7583</td>\n",
       "      <td>0.0</td>\n",
       "      <td>187.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>613</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4583</td>\n",
       "      <td>0.0</td>\n",
       "      <td>133.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>601 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Gender  Married Dependents  Education Self_Employed  ApplicantIncome  \\\n",
       "0        1        0          0          0             0             5849   \n",
       "1        1        1          1          0             0             4583   \n",
       "2        1        1          0          0             1             3000   \n",
       "3        1        1          0          1             0             2583   \n",
       "4        1        0          0          0             0             6000   \n",
       "..     ...      ...        ...        ...           ...              ...   \n",
       "609      0        0          0          0             0             2900   \n",
       "610      1        1          3          0             0             4106   \n",
       "611      1        1          1          0             0             8072   \n",
       "612      1        1          2          0             0             7583   \n",
       "613      0        0          0          0             1             4583   \n",
       "\n",
       "     CoapplicantIncome  LoanAmount  Loan_Amount_Term  Credit_History  \\\n",
       "0                  0.0         NaN             360.0             1.0   \n",
       "1               1508.0       128.0             360.0             1.0   \n",
       "2                  0.0        66.0             360.0             1.0   \n",
       "3               2358.0       120.0             360.0             1.0   \n",
       "4                  0.0       141.0             360.0             1.0   \n",
       "..                 ...         ...               ...             ...   \n",
       "609                0.0        71.0             360.0             1.0   \n",
       "610                0.0        40.0             180.0             1.0   \n",
       "611              240.0       253.0             360.0             1.0   \n",
       "612                0.0       187.0             360.0             1.0   \n",
       "613                0.0       133.0             360.0             0.0   \n",
       "\n",
       "     Property_Area Loan_Status  \n",
       "0                2           Y  \n",
       "1                0           N  \n",
       "2                2           Y  \n",
       "3                2           Y  \n",
       "4                2           Y  \n",
       "..             ...         ...  \n",
       "609              0           Y  \n",
       "610              0           Y  \n",
       "611              2           Y  \n",
       "612              2           Y  \n",
       "613              1           N  \n",
       "\n",
       "[601 rows x 12 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ordinal_encoder = OrdinalEncoder()\n",
    "ordinal_encoder.categories_ = np.load('saves/variable_encoder_categories.npy', allow_pickle= True)\n",
    "categorical_columns = ['Gender', 'Married', 'Dependents', 'Education', 'Self_Employed', 'Property_Area','Credit_History','Loan_Amount_Term','Loan_Status']\n",
    "df_encoded_nans = preprocess_utils.encode_with_nan(df_rows, categorical_columns, ordinal_encoder)\n",
    "df_encoded_nans"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Although Credit_History and Loan_Amount_Term are represented as numerical values, we will set them as categorical variables. \n",
    "\n",
    "That is because these variables have a categorical behavior, as shown below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column Values:\n",
      "Credit_History : [ 1.  0. nan]\n",
      "Loan_Amount_Term : [360. 120. 240.  nan 180.  60. 300. 480.  36.  84.  12.]\n"
     ]
    }
   ],
   "source": [
    "dataframe_utils.show_column_options(df_import[['Credit_History','Loan_Amount_Term']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MissForest(bootstrap=True, class_weight='balanced', copy=True,\n",
       "           criterion=('mse', 'gini'), decreasing=False, max_depth=None,\n",
       "           max_features='auto', max_iter=10, max_leaf_nodes=None,\n",
       "           min_impurity_decrease=0.0, min_samples_leaf=1, min_samples_split=2,\n",
       "           min_weight_fraction_leaf=0.0, missing_values=nan, n_estimators=100,\n",
       "           n_jobs=-1, oob_score=True, random_state=0, verbose=0,\n",
       "           warm_start=False)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = df_encoded_nans.copy().drop(columns = ['Loan_Status'])\n",
    "y = df_encoded_nans.copy()[['Loan_Status']]\n",
    "\n",
    "\n",
    "categorical_index = [0,1,2,3,4,8,9,10]\n",
    "imputer = MissForest(oob_score=True, random_state = 0, class_weight = 'balanced')\n",
    "imputer.fit(X,y,cat_vars = categorical_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 0\n",
      "Iteration: 1\n",
      "Iteration: 2\n",
      "Iteration: 3\n",
      "Iteration: 4\n"
     ]
    }
   ],
   "source": [
    "X_filled = imputer.transform(X)\n",
    "df_fill = pd.DataFrame(X_filled,columns = X.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gender</th>\n",
       "      <th>Married</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>Education</th>\n",
       "      <th>Self_Employed</th>\n",
       "      <th>ApplicantIncome</th>\n",
       "      <th>CoapplicantIncome</th>\n",
       "      <th>LoanAmount</th>\n",
       "      <th>Loan_Amount_Term</th>\n",
       "      <th>Credit_History</th>\n",
       "      <th>Property_Area</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5849.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>145.09</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4583.0</td>\n",
       "      <td>1508.0</td>\n",
       "      <td>128.00</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66.00</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2583.0</td>\n",
       "      <td>2358.0</td>\n",
       "      <td>120.00</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>141.00</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>596</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2900.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.00</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>597</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4106.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40.00</td>\n",
       "      <td>180.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>598</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8072.0</td>\n",
       "      <td>240.0</td>\n",
       "      <td>253.00</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7583.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>187.00</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>600</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4583.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>133.00</td>\n",
       "      <td>360.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>601 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Gender  Married  Dependents  Education  Self_Employed  ApplicantIncome  \\\n",
       "0       1.0      0.0         0.0        0.0            0.0           5849.0   \n",
       "1       1.0      1.0         1.0        0.0            0.0           4583.0   \n",
       "2       1.0      1.0         0.0        0.0            1.0           3000.0   \n",
       "3       1.0      1.0         0.0        1.0            0.0           2583.0   \n",
       "4       1.0      0.0         0.0        0.0            0.0           6000.0   \n",
       "..      ...      ...         ...        ...            ...              ...   \n",
       "596     0.0      0.0         0.0        0.0            0.0           2900.0   \n",
       "597     1.0      1.0         3.0        0.0            0.0           4106.0   \n",
       "598     1.0      1.0         1.0        0.0            0.0           8072.0   \n",
       "599     1.0      1.0         2.0        0.0            0.0           7583.0   \n",
       "600     0.0      0.0         0.0        0.0            1.0           4583.0   \n",
       "\n",
       "     CoapplicantIncome  LoanAmount  Loan_Amount_Term  Credit_History  \\\n",
       "0                  0.0      145.09             360.0             1.0   \n",
       "1               1508.0      128.00             360.0             1.0   \n",
       "2                  0.0       66.00             360.0             1.0   \n",
       "3               2358.0      120.00             360.0             1.0   \n",
       "4                  0.0      141.00             360.0             1.0   \n",
       "..                 ...         ...               ...             ...   \n",
       "596                0.0       71.00             360.0             1.0   \n",
       "597                0.0       40.00             180.0             1.0   \n",
       "598              240.0      253.00             360.0             1.0   \n",
       "599                0.0      187.00             360.0             1.0   \n",
       "600                0.0      133.00             360.0             0.0   \n",
       "\n",
       "     Property_Area  \n",
       "0              2.0  \n",
       "1              0.0  \n",
       "2              2.0  \n",
       "3              2.0  \n",
       "4              2.0  \n",
       "..             ...  \n",
       "596            0.0  \n",
       "597            0.0  \n",
       "598            2.0  \n",
       "599            2.0  \n",
       "600            1.0  \n",
       "\n",
       "[601 rows x 11 columns]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_fill"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fill['Loan_Status'] = y.values.ravel()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pre processing after Imputation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculating Base_Loan_Installment and Remaining_Income"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_loan_installment = df_fill['LoanAmount'] * 1000 / df_fill['Loan_Amount_Term']\n",
    "total_income = df_fill['ApplicantIncome'] + df_fill['CoapplicantIncome']\n",
    "remaining_income = (total_income - base_loan_installment) / total_income\n",
    "\n",
    "df_fill['Base_Loan_Installment'] = base_loan_installment\n",
    "df_fill['Remaining_Income'] = remaining_income"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Removing outliers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(570, 14)\n"
     ]
    }
   ],
   "source": [
    "numerical_columns = ['ApplicantIncome', 'CoapplicantIncome', 'LoanAmount','Base_Loan_Installment','Remaining_Income']\n",
    "df_fill_no_outlier = preprocess_utils.remove_outliers(df_fill,numerical_columns,threshold = 3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Envoding Loan_Status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fill_no_outlier.loc[df_fill_no_outlier['Loan_Status'] == 'Y','Loan_Status'] = 1\n",
    "df_fill_no_outlier.loc[df_fill_no_outlier['Loan_Status'] == 'N','Loan_Status'] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fill_no_outlier.to_csv('dataset/train_rf_imputed.csv',index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}