1. Model_Training.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pycaret.classification import *\n",
    "import os\n",
    "from sklearn.metrics import average_precision_score\n",
    "from sklearn.utils import shuffle\n",
    "from model_utils import *\n",
    "import mlflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlflow.set_tracking_uri(\"http://localhost:5000\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Read the Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Time</th>\n",
       "      <th>V1</th>\n",
       "      <th>V2</th>\n",
       "      <th>V3</th>\n",
       "      <th>V4</th>\n",
       "      <th>V5</th>\n",
       "      <th>V6</th>\n",
       "      <th>V7</th>\n",
       "      <th>V8</th>\n",
       "      <th>V9</th>\n",
       "      <th>V10</th>\n",
       "      <th>V11</th>\n",
       "      <th>V12</th>\n",
       "      <th>V13</th>\n",
       "      <th>V14</th>\n",
       "      <th>V15</th>\n",
       "      <th>V16</th>\n",
       "      <th>V17</th>\n",
       "      <th>V18</th>\n",
       "      <th>V19</th>\n",
       "      <th>V20</th>\n",
       "      <th>V21</th>\n",
       "      <th>V22</th>\n",
       "      <th>V23</th>\n",
       "      <th>V24</th>\n",
       "      <th>V25</th>\n",
       "      <th>V26</th>\n",
       "      <th>V27</th>\n",
       "      <th>V28</th>\n",
       "      <th>Amount</th>\n",
       "      <th>Class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.359807</td>\n",
       "      <td>-0.072781</td>\n",
       "      <td>2.536347</td>\n",
       "      <td>1.378155</td>\n",
       "      <td>-0.338321</td>\n",
       "      <td>0.462388</td>\n",
       "      <td>0.239599</td>\n",
       "      <td>0.098698</td>\n",
       "      <td>0.363787</td>\n",
       "      <td>0.090794</td>\n",
       "      <td>-0.551600</td>\n",
       "      <td>-0.617801</td>\n",
       "      <td>-0.991390</td>\n",
       "      <td>-0.311169</td>\n",
       "      <td>1.468177</td>\n",
       "      <td>-0.470401</td>\n",
       "      <td>0.207971</td>\n",
       "      <td>0.025791</td>\n",
       "      <td>0.403993</td>\n",
       "      <td>0.251412</td>\n",
       "      <td>-0.018307</td>\n",
       "      <td>0.277838</td>\n",
       "      <td>-0.110474</td>\n",
       "      <td>0.066928</td>\n",
       "      <td>0.128539</td>\n",
       "      <td>-0.189115</td>\n",
       "      <td>0.133558</td>\n",
       "      <td>-0.021053</td>\n",
       "      <td>149.62</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.191857</td>\n",
       "      <td>0.266151</td>\n",
       "      <td>0.166480</td>\n",
       "      <td>0.448154</td>\n",
       "      <td>0.060018</td>\n",
       "      <td>-0.082361</td>\n",
       "      <td>-0.078803</td>\n",
       "      <td>0.085102</td>\n",
       "      <td>-0.255425</td>\n",
       "      <td>-0.166974</td>\n",
       "      <td>1.612727</td>\n",
       "      <td>1.065235</td>\n",
       "      <td>0.489095</td>\n",
       "      <td>-0.143772</td>\n",
       "      <td>0.635558</td>\n",
       "      <td>0.463917</td>\n",
       "      <td>-0.114805</td>\n",
       "      <td>-0.183361</td>\n",
       "      <td>-0.145783</td>\n",
       "      <td>-0.069083</td>\n",
       "      <td>-0.225775</td>\n",
       "      <td>-0.638672</td>\n",
       "      <td>0.101288</td>\n",
       "      <td>-0.339846</td>\n",
       "      <td>0.167170</td>\n",
       "      <td>0.125895</td>\n",
       "      <td>-0.008983</td>\n",
       "      <td>0.014724</td>\n",
       "      <td>2.69</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.358354</td>\n",
       "      <td>-1.340163</td>\n",
       "      <td>1.773209</td>\n",
       "      <td>0.379780</td>\n",
       "      <td>-0.503198</td>\n",
       "      <td>1.800499</td>\n",
       "      <td>0.791461</td>\n",
       "      <td>0.247676</td>\n",
       "      <td>-1.514654</td>\n",
       "      <td>0.207643</td>\n",
       "      <td>0.624501</td>\n",
       "      <td>0.066084</td>\n",
       "      <td>0.717293</td>\n",
       "      <td>-0.165946</td>\n",
       "      <td>2.345865</td>\n",
       "      <td>-2.890083</td>\n",
       "      <td>1.109969</td>\n",
       "      <td>-0.121359</td>\n",
       "      <td>-2.261857</td>\n",
       "      <td>0.524980</td>\n",
       "      <td>0.247998</td>\n",
       "      <td>0.771679</td>\n",
       "      <td>0.909412</td>\n",
       "      <td>-0.689281</td>\n",
       "      <td>-0.327642</td>\n",
       "      <td>-0.139097</td>\n",
       "      <td>-0.055353</td>\n",
       "      <td>-0.059752</td>\n",
       "      <td>378.66</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.966272</td>\n",
       "      <td>-0.185226</td>\n",
       "      <td>1.792993</td>\n",
       "      <td>-0.863291</td>\n",
       "      <td>-0.010309</td>\n",
       "      <td>1.247203</td>\n",
       "      <td>0.237609</td>\n",
       "      <td>0.377436</td>\n",
       "      <td>-1.387024</td>\n",
       "      <td>-0.054952</td>\n",
       "      <td>-0.226487</td>\n",
       "      <td>0.178228</td>\n",
       "      <td>0.507757</td>\n",
       "      <td>-0.287924</td>\n",
       "      <td>-0.631418</td>\n",
       "      <td>-1.059647</td>\n",
       "      <td>-0.684093</td>\n",
       "      <td>1.965775</td>\n",
       "      <td>-1.232622</td>\n",
       "      <td>-0.208038</td>\n",
       "      <td>-0.108300</td>\n",
       "      <td>0.005274</td>\n",
       "      <td>-0.190321</td>\n",
       "      <td>-1.175575</td>\n",
       "      <td>0.647376</td>\n",
       "      <td>-0.221929</td>\n",
       "      <td>0.062723</td>\n",
       "      <td>0.061458</td>\n",
       "      <td>123.50</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.0</td>\n",
       "      <td>-1.158233</td>\n",
       "      <td>0.877737</td>\n",
       "      <td>1.548718</td>\n",
       "      <td>0.403034</td>\n",
       "      <td>-0.407193</td>\n",
       "      <td>0.095921</td>\n",
       "      <td>0.592941</td>\n",
       "      <td>-0.270533</td>\n",
       "      <td>0.817739</td>\n",
       "      <td>0.753074</td>\n",
       "      <td>-0.822843</td>\n",
       "      <td>0.538196</td>\n",
       "      <td>1.345852</td>\n",
       "      <td>-1.119670</td>\n",
       "      <td>0.175121</td>\n",
       "      <td>-0.451449</td>\n",
       "      <td>-0.237033</td>\n",
       "      <td>-0.038195</td>\n",
       "      <td>0.803487</td>\n",
       "      <td>0.408542</td>\n",
       "      <td>-0.009431</td>\n",
       "      <td>0.798278</td>\n",
       "      <td>-0.137458</td>\n",
       "      <td>0.141267</td>\n",
       "      <td>-0.206010</td>\n",
       "      <td>0.502292</td>\n",
       "      <td>0.219422</td>\n",
       "      <td>0.215153</td>\n",
       "      <td>69.99</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Time        V1        V2        V3        V4        V5        V6        V7  \\\n",
       "0   0.0 -1.359807 -0.072781  2.536347  1.378155 -0.338321  0.462388  0.239599   \n",
       "1   0.0  1.191857  0.266151  0.166480  0.448154  0.060018 -0.082361 -0.078803   \n",
       "2   1.0 -1.358354 -1.340163  1.773209  0.379780 -0.503198  1.800499  0.791461   \n",
       "3   1.0 -0.966272 -0.185226  1.792993 -0.863291 -0.010309  1.247203  0.237609   \n",
       "4   2.0 -1.158233  0.877737  1.548718  0.403034 -0.407193  0.095921  0.592941   \n",
       "\n",
       "         V8        V9       V10       V11       V12       V13       V14  \\\n",
       "0  0.098698  0.363787  0.090794 -0.551600 -0.617801 -0.991390 -0.311169   \n",
       "1  0.085102 -0.255425 -0.166974  1.612727  1.065235  0.489095 -0.143772   \n",
       "2  0.247676 -1.514654  0.207643  0.624501  0.066084  0.717293 -0.165946   \n",
       "3  0.377436 -1.387024 -0.054952 -0.226487  0.178228  0.507757 -0.287924   \n",
       "4 -0.270533  0.817739  0.753074 -0.822843  0.538196  1.345852 -1.119670   \n",
       "\n",
       "        V15       V16       V17       V18       V19       V20       V21  \\\n",
       "0  1.468177 -0.470401  0.207971  0.025791  0.403993  0.251412 -0.018307   \n",
       "1  0.635558  0.463917 -0.114805 -0.183361 -0.145783 -0.069083 -0.225775   \n",
       "2  2.345865 -2.890083  1.109969 -0.121359 -2.261857  0.524980  0.247998   \n",
       "3 -0.631418 -1.059647 -0.684093  1.965775 -1.232622 -0.208038 -0.108300   \n",
       "4  0.175121 -0.451449 -0.237033 -0.038195  0.803487  0.408542 -0.009431   \n",
       "\n",
       "        V22       V23       V24       V25       V26       V27       V28  \\\n",
       "0  0.277838 -0.110474  0.066928  0.128539 -0.189115  0.133558 -0.021053   \n",
       "1 -0.638672  0.101288 -0.339846  0.167170  0.125895 -0.008983  0.014724   \n",
       "2  0.771679  0.909412 -0.689281 -0.327642 -0.139097 -0.055353 -0.059752   \n",
       "3  0.005274 -0.190321 -1.175575  0.647376 -0.221929  0.062723  0.061458   \n",
       "4  0.798278 -0.137458  0.141267 -0.206010  0.502292  0.219422  0.215153   \n",
       "\n",
       "   Amount  Class  \n",
       "0  149.62      0  \n",
       "1    2.69      0  \n",
       "2  378.66      0  \n",
       "3  123.50      0  \n",
       "4   69.99      0  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('dataset' + os.sep + 'creditcard.csv', encoding_errors='ignore', on_bad_lines='skip')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    284315\n",
       "1       492\n",
       "Name: Class, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.Class.value_counts()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Split the data into train and test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train, df_test = get_raw_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0    1916\n",
       " 1     321\n",
       " Name: target, dtype: int64,\n",
       " 0    932\n",
       " 1    171\n",
       " Name: target, dtype: int64)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.target.value_counts(), df_test.target.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Time', 'V1', 'V2', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10',\n",
       "       'V11', 'V12', 'V13', 'V14', 'V15', 'V16', 'V17', 'V18', 'V19', 'V20',\n",
       "       'V21', 'V22', 'V23', 'V24', 'V25', 'V26', 'V27', 'V28', 'Amount',\n",
       "       'target'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.columns"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Training different models with different data preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Time',\n",
       " 'V1',\n",
       " 'V2',\n",
       " 'V3',\n",
       " 'V4',\n",
       " 'V5',\n",
       " 'V6',\n",
       " 'V7',\n",
       " 'V8',\n",
       " 'V9',\n",
       " 'V10',\n",
       " 'V11',\n",
       " 'V12',\n",
       " 'V13',\n",
       " 'V14',\n",
       " 'V15',\n",
       " 'V16',\n",
       " 'V17',\n",
       " 'V18',\n",
       " 'V19',\n",
       " 'V20',\n",
       " 'V21',\n",
       " 'V22',\n",
       " 'V23',\n",
       " 'V24',\n",
       " 'V25',\n",
       " 'V26',\n",
       " 'V27',\n",
       " 'V28',\n",
       " 'Amount']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "numeric_feature_list = df_train.columns[0:-1].to_list()\n",
    "numeric_feature_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf1 = setup(silent=True,\n",
    "             data = df_train, \n",
    "             target = 'target',\n",
    "             numeric_features=numeric_feature_list,\n",
    "             log_experiment = True,\n",
    "             use_gpu=False,\n",
    "             experiment_name = 'baseline')\n",
    "\n",
    "add_metric('apc', 'APC', average_precision_score, target = 'pred_proba')\n",
    "best = compare_models(sort=\"APC\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Fixing imbalance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_e8968 th {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_e8968_row0_col0, #T_e8968_row0_col1, #T_e8968_row0_col3, #T_e8968_row0_col4, #T_e8968_row0_col5, #T_e8968_row0_col6, #T_e8968_row0_col7, #T_e8968_row1_col0, #T_e8968_row1_col1, #T_e8968_row1_col2, #T_e8968_row1_col4, #T_e8968_row1_col5, #T_e8968_row1_col6, #T_e8968_row1_col7, #T_e8968_row1_col8, #T_e8968_row2_col0, #T_e8968_row2_col2, #T_e8968_row2_col3, #T_e8968_row2_col8, #T_e8968_row3_col0, #T_e8968_row3_col1, #T_e8968_row3_col2, #T_e8968_row3_col3, #T_e8968_row3_col4, #T_e8968_row3_col5, #T_e8968_row3_col6, #T_e8968_row3_col7, #T_e8968_row3_col8, #T_e8968_row4_col0, #T_e8968_row4_col1, #T_e8968_row4_col2, #T_e8968_row4_col3, #T_e8968_row4_col4, #T_e8968_row4_col5, #T_e8968_row4_col6, #T_e8968_row4_col7, #T_e8968_row4_col8, #T_e8968_row5_col0, #T_e8968_row5_col1, #T_e8968_row5_col2, #T_e8968_row5_col3, #T_e8968_row5_col4, #T_e8968_row5_col5, #T_e8968_row5_col6, #T_e8968_row5_col7, #T_e8968_row5_col8, #T_e8968_row6_col0, #T_e8968_row6_col1, #T_e8968_row6_col2, #T_e8968_row6_col3, #T_e8968_row6_col4, #T_e8968_row6_col5, #T_e8968_row6_col6, #T_e8968_row6_col7, #T_e8968_row6_col8, #T_e8968_row7_col0, #T_e8968_row7_col1, #T_e8968_row7_col2, #T_e8968_row7_col4, #T_e8968_row7_col5, #T_e8968_row7_col6, #T_e8968_row7_col7, #T_e8968_row7_col8, #T_e8968_row8_col0, #T_e8968_row8_col1, #T_e8968_row8_col2, #T_e8968_row8_col3, #T_e8968_row8_col4, #T_e8968_row8_col5, #T_e8968_row8_col6, #T_e8968_row8_col7, #T_e8968_row8_col8, #T_e8968_row9_col0, #T_e8968_row9_col1, #T_e8968_row9_col2, #T_e8968_row9_col3, #T_e8968_row9_col4, #T_e8968_row9_col5, #T_e8968_row9_col6, #T_e8968_row9_col7, #T_e8968_row9_col8, #T_e8968_row10_col0, #T_e8968_row10_col1, #T_e8968_row10_col2, #T_e8968_row10_col3, #T_e8968_row10_col4, #T_e8968_row10_col5, #T_e8968_row10_col6, #T_e8968_row10_col7, #T_e8968_row10_col8, #T_e8968_row11_col0, #T_e8968_row11_col1, #T_e8968_row11_col2, #T_e8968_row11_col3, #T_e8968_row11_col4, #T_e8968_row11_col5, #T_e8968_row11_col6, #T_e8968_row11_col7, #T_e8968_row11_col8, #T_e8968_row12_col0, #T_e8968_row12_col1, #T_e8968_row12_col2, #T_e8968_row12_col3, #T_e8968_row12_col4, #T_e8968_row12_col5, #T_e8968_row12_col6, #T_e8968_row12_col7, #T_e8968_row12_col8 {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_e8968_row0_col2, #T_e8968_row0_col8, #T_e8968_row1_col3, #T_e8968_row2_col1, #T_e8968_row2_col4, #T_e8968_row2_col5, #T_e8968_row2_col6, #T_e8968_row2_col7, #T_e8968_row7_col3 {\n",
       "  text-align: left;\n",
       "  background-color: yellow;\n",
       "}\n",
       "#T_e8968_row0_col9, #T_e8968_row1_col9, #T_e8968_row2_col9, #T_e8968_row3_col9, #T_e8968_row4_col9, #T_e8968_row5_col9, #T_e8968_row6_col9, #T_e8968_row7_col9, #T_e8968_row8_col9, #T_e8968_row9_col9, #T_e8968_row10_col9, #T_e8968_row11_col9 {\n",
       "  text-align: left;\n",
       "  background-color: lightgrey;\n",
       "}\n",
       "#T_e8968_row12_col9 {\n",
       "  text-align: left;\n",
       "  background-color: yellow;\n",
       "  background-color: lightgrey;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_e8968\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_e8968_level0_col0\" class=\"col_heading level0 col0\" >Model</th>\n",
       "      <th id=\"T_e8968_level0_col1\" class=\"col_heading level0 col1\" >Accuracy</th>\n",
       "      <th id=\"T_e8968_level0_col2\" class=\"col_heading level0 col2\" >AUC</th>\n",
       "      <th id=\"T_e8968_level0_col3\" class=\"col_heading level0 col3\" >Recall</th>\n",
       "      <th id=\"T_e8968_level0_col4\" class=\"col_heading level0 col4\" >Prec.</th>\n",
       "      <th id=\"T_e8968_level0_col5\" class=\"col_heading level0 col5\" >F1</th>\n",
       "      <th id=\"T_e8968_level0_col6\" class=\"col_heading level0 col6\" >Kappa</th>\n",
       "      <th id=\"T_e8968_level0_col7\" class=\"col_heading level0 col7\" >MCC</th>\n",
       "      <th id=\"T_e8968_level0_col8\" class=\"col_heading level0 col8\" >APC</th>\n",
       "      <th id=\"T_e8968_level0_col9\" class=\"col_heading level0 col9\" >TT (Sec)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_e8968_level0_row0\" class=\"row_heading level0 row0\" >rf</th>\n",
       "      <td id=\"T_e8968_row0_col0\" class=\"data row0 col0\" >Random Forest Classifier</td>\n",
       "      <td id=\"T_e8968_row0_col1\" class=\"data row0 col1\" >0.9789</td>\n",
       "      <td id=\"T_e8968_row0_col2\" class=\"data row0 col2\" >0.9822</td>\n",
       "      <td id=\"T_e8968_row0_col3\" class=\"data row0 col3\" >0.8845</td>\n",
       "      <td id=\"T_e8968_row0_col4\" class=\"data row0 col4\" >0.9788</td>\n",
       "      <td id=\"T_e8968_row0_col5\" class=\"data row0 col5\" >0.9287</td>\n",
       "      <td id=\"T_e8968_row0_col6\" class=\"data row0 col6\" >0.9163</td>\n",
       "      <td id=\"T_e8968_row0_col7\" class=\"data row0 col7\" >0.9183</td>\n",
       "      <td id=\"T_e8968_row0_col8\" class=\"data row0 col8\" >0.9588</td>\n",
       "      <td id=\"T_e8968_row0_col9\" class=\"data row0 col9\" >4.4060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_e8968_level0_row1\" class=\"row_heading level0 row1\" >gbc</th>\n",
       "      <td id=\"T_e8968_row1_col0\" class=\"data row1 col0\" >Gradient Boosting Classifier</td>\n",
       "      <td id=\"T_e8968_row1_col1\" class=\"data row1 col1\" >0.9770</td>\n",
       "      <td id=\"T_e8968_row1_col2\" class=\"data row1 col2\" >0.9754</td>\n",
       "      <td id=\"T_e8968_row1_col3\" class=\"data row1 col3\" >0.9012</td>\n",
       "      <td id=\"T_e8968_row1_col4\" class=\"data row1 col4\" >0.9500</td>\n",
       "      <td id=\"T_e8968_row1_col5\" class=\"data row1 col5\" >0.9236</td>\n",
       "      <td id=\"T_e8968_row1_col6\" class=\"data row1 col6\" >0.9101</td>\n",
       "      <td id=\"T_e8968_row1_col7\" class=\"data row1 col7\" >0.9114</td>\n",
       "      <td id=\"T_e8968_row1_col8\" class=\"data row1 col8\" >0.9561</td>\n",
       "      <td id=\"T_e8968_row1_col9\" class=\"data row1 col9\" >14.0630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_e8968_level0_row2\" class=\"row_heading level0 row2\" >et</th>\n",
       "      <td id=\"T_e8968_row2_col0\" class=\"data row2 col0\" >Extra Trees Classifier</td>\n",
       "      <td id=\"T_e8968_row2_col1\" class=\"data row2 col1\" >0.9802</td>\n",
       "      <td id=\"T_e8968_row2_col2\" class=\"data row2 col2\" >0.9781</td>\n",
       "      <td id=\"T_e8968_row2_col3\" class=\"data row2 col3\" >0.8845</td>\n",
       "      <td id=\"T_e8968_row2_col4\" class=\"data row2 col4\" >0.9868</td>\n",
       "      <td id=\"T_e8968_row2_col5\" class=\"data row2 col5\" >0.9322</td>\n",
       "      <td id=\"T_e8968_row2_col6\" class=\"data row2 col6\" >0.9207</td>\n",
       "      <td id=\"T_e8968_row2_col7\" class=\"data row2 col7\" >0.9230</td>\n",
       "      <td id=\"T_e8968_row2_col8\" class=\"data row2 col8\" >0.9558</td>\n",
       "      <td id=\"T_e8968_row2_col9\" class=\"data row2 col9\" >2.8120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_e8968_level0_row3\" class=\"row_heading level0 row3\" >ada</th>\n",
       "      <td id=\"T_e8968_row3_col0\" class=\"data row3 col0\" >Ada Boost Classifier</td>\n",
       "      <td id=\"T_e8968_row3_col1\" class=\"data row3 col1\" >0.9674</td>\n",
       "      <td id=\"T_e8968_row3_col2\" class=\"data row3 col2\" >0.9739</td>\n",
       "      <td id=\"T_e8968_row3_col3\" class=\"data row3 col3\" >0.8885</td>\n",
       "      <td id=\"T_e8968_row3_col4\" class=\"data row3 col4\" >0.9022</td>\n",
       "      <td id=\"T_e8968_row3_col5\" class=\"data row3 col5\" >0.8948</td>\n",
       "      <td id=\"T_e8968_row3_col6\" class=\"data row3 col6\" >0.8755</td>\n",
       "      <td id=\"T_e8968_row3_col7\" class=\"data row3 col7\" >0.8759</td>\n",
       "      <td id=\"T_e8968_row3_col8\" class=\"data row3 col8\" >0.9457</td>\n",
       "      <td id=\"T_e8968_row3_col9\" class=\"data row3 col9\" >4.1540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_e8968_level0_row4\" class=\"row_heading level0 row4\" >lr</th>\n",
       "      <td id=\"T_e8968_row4_col0\" class=\"data row4 col0\" >Logistic Regression</td>\n",
       "      <td id=\"T_e8968_row4_col1\" class=\"data row4 col1\" >0.9680</td>\n",
       "      <td id=\"T_e8968_row4_col2\" class=\"data row4 col2\" >0.9686</td>\n",
       "      <td id=\"T_e8968_row4_col3\" class=\"data row4 col3\" >0.8927</td>\n",
       "      <td id=\"T_e8968_row4_col4\" class=\"data row4 col4\" >0.9028</td>\n",
       "      <td id=\"T_e8968_row4_col5\" class=\"data row4 col5\" >0.8967</td>\n",
       "      <td id=\"T_e8968_row4_col6\" class=\"data row4 col6\" >0.8778</td>\n",
       "      <td id=\"T_e8968_row4_col7\" class=\"data row4 col7\" >0.8785</td>\n",
       "      <td id=\"T_e8968_row4_col8\" class=\"data row4 col8\" >0.9452</td>\n",
       "      <td id=\"T_e8968_row4_col9\" class=\"data row4 col9\" >0.9840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_e8968_level0_row5\" class=\"row_heading level0 row5\" >lda</th>\n",
       "      <td id=\"T_e8968_row5_col0\" class=\"data row5 col0\" >Linear Discriminant Analysis</td>\n",
       "      <td id=\"T_e8968_row5_col1\" class=\"data row5 col1\" >0.9655</td>\n",
       "      <td id=\"T_e8968_row5_col2\" class=\"data row5 col2\" >0.9691</td>\n",
       "      <td id=\"T_e8968_row5_col3\" class=\"data row5 col3\" >0.8263</td>\n",
       "      <td id=\"T_e8968_row5_col4\" class=\"data row5 col4\" >0.9439</td>\n",
       "      <td id=\"T_e8968_row5_col5\" class=\"data row5 col5\" >0.8801</td>\n",
       "      <td id=\"T_e8968_row5_col6\" class=\"data row5 col6\" >0.8601</td>\n",
       "      <td id=\"T_e8968_row5_col7\" class=\"data row5 col7\" >0.8634</td>\n",
       "      <td id=\"T_e8968_row5_col8\" class=\"data row5 col8\" >0.9245</td>\n",
       "      <td id=\"T_e8968_row5_col9\" class=\"data row5 col9\" >0.2180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_e8968_level0_row6\" class=\"row_heading level0 row6\" >nb</th>\n",
       "      <td id=\"T_e8968_row6_col0\" class=\"data row6 col0\" >Naive Bayes</td>\n",
       "      <td id=\"T_e8968_row6_col1\" class=\"data row6 col1\" >0.9578</td>\n",
       "      <td id=\"T_e8968_row6_col2\" class=\"data row6 col2\" >0.9680</td>\n",
       "      <td id=\"T_e8968_row6_col3\" class=\"data row6 col3\" >0.7852</td>\n",
       "      <td id=\"T_e8968_row6_col4\" class=\"data row6 col4\" >0.9382</td>\n",
       "      <td id=\"T_e8968_row6_col5\" class=\"data row6 col5\" >0.8512</td>\n",
       "      <td id=\"T_e8968_row6_col6\" class=\"data row6 col6\" >0.8269</td>\n",
       "      <td id=\"T_e8968_row6_col7\" class=\"data row6 col7\" >0.8337</td>\n",
       "      <td id=\"T_e8968_row6_col8\" class=\"data row6 col8\" >0.9161</td>\n",
       "      <td id=\"T_e8968_row6_col9\" class=\"data row6 col9\" >0.2240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_e8968_level0_row7\" class=\"row_heading level0 row7\" >qda</th>\n",
       "      <td id=\"T_e8968_row7_col0\" class=\"data row7 col0\" >Quadratic Discriminant Analysis</td>\n",
       "      <td id=\"T_e8968_row7_col1\" class=\"data row7 col1\" >0.9489</td>\n",
       "      <td id=\"T_e8968_row7_col2\" class=\"data row7 col2\" >0.9644</td>\n",
       "      <td id=\"T_e8968_row7_col3\" class=\"data row7 col3\" >0.9012</td>\n",
       "      <td id=\"T_e8968_row7_col4\" class=\"data row7 col4\" >0.7993</td>\n",
       "      <td id=\"T_e8968_row7_col5\" class=\"data row7 col5\" >0.8460</td>\n",
       "      <td id=\"T_e8968_row7_col6\" class=\"data row7 col6\" >0.8155</td>\n",
       "      <td id=\"T_e8968_row7_col7\" class=\"data row7 col7\" >0.8185</td>\n",
       "      <td id=\"T_e8968_row7_col8\" class=\"data row7 col8\" >0.8775</td>\n",
       "      <td id=\"T_e8968_row7_col9\" class=\"data row7 col9\" >0.2400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_e8968_level0_row8\" class=\"row_heading level0 row8\" >dt</th>\n",
       "      <td id=\"T_e8968_row8_col0\" class=\"data row8 col0\" >Decision Tree Classifier</td>\n",
       "      <td id=\"T_e8968_row8_col1\" class=\"data row8 col1\" >0.9559</td>\n",
       "      <td id=\"T_e8968_row8_col2\" class=\"data row8 col2\" >0.9249</td>\n",
       "      <td id=\"T_e8968_row8_col3\" class=\"data row8 col3\" >0.8800</td>\n",
       "      <td id=\"T_e8968_row8_col4\" class=\"data row8 col4\" >0.8462</td>\n",
       "      <td id=\"T_e8968_row8_col5\" class=\"data row8 col5\" >0.8609</td>\n",
       "      <td id=\"T_e8968_row8_col6\" class=\"data row8 col6\" >0.8348</td>\n",
       "      <td id=\"T_e8968_row8_col7\" class=\"data row8 col7\" >0.8363</td>\n",
       "      <td id=\"T_e8968_row8_col8\" class=\"data row8 col8\" >0.7631</td>\n",
       "      <td id=\"T_e8968_row8_col9\" class=\"data row8 col9\" >0.7760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_e8968_level0_row9\" class=\"row_heading level0 row9\" >knn</th>\n",
       "      <td id=\"T_e8968_row9_col0\" class=\"data row9 col0\" >K Neighbors Classifier</td>\n",
       "      <td id=\"T_e8968_row9_col1\" class=\"data row9 col1\" >0.6927</td>\n",
       "      <td id=\"T_e8968_row9_col2\" class=\"data row9 col2\" >0.6711</td>\n",
       "      <td id=\"T_e8968_row9_col3\" class=\"data row9 col3\" >0.5498</td>\n",
       "      <td id=\"T_e8968_row9_col4\" class=\"data row9 col4\" >0.2636</td>\n",
       "      <td id=\"T_e8968_row9_col5\" class=\"data row9 col5\" >0.3551</td>\n",
       "      <td id=\"T_e8968_row9_col6\" class=\"data row9 col6\" >0.1855</td>\n",
       "      <td id=\"T_e8968_row9_col7\" class=\"data row9 col7\" >0.2081</td>\n",
       "      <td id=\"T_e8968_row9_col8\" class=\"data row9 col8\" >0.2794</td>\n",
       "      <td id=\"T_e8968_row9_col9\" class=\"data row9 col9\" >0.4930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_e8968_level0_row10\" class=\"row_heading level0 row10\" >dummy</th>\n",
       "      <td id=\"T_e8968_row10_col0\" class=\"data row10 col0\" >Dummy Classifier</td>\n",
       "      <td id=\"T_e8968_row10_col1\" class=\"data row10 col1\" >0.8454</td>\n",
       "      <td id=\"T_e8968_row10_col2\" class=\"data row10 col2\" >0.5000</td>\n",
       "      <td id=\"T_e8968_row10_col3\" class=\"data row10 col3\" >0.0000</td>\n",
       "      <td id=\"T_e8968_row10_col4\" class=\"data row10 col4\" >0.0000</td>\n",
       "      <td id=\"T_e8968_row10_col5\" class=\"data row10 col5\" >0.0000</td>\n",
       "      <td id=\"T_e8968_row10_col6\" class=\"data row10 col6\" >0.0000</td>\n",
       "      <td id=\"T_e8968_row10_col7\" class=\"data row10 col7\" >0.0000</td>\n",
       "      <td id=\"T_e8968_row10_col8\" class=\"data row10 col8\" >0.1546</td>\n",
       "      <td id=\"T_e8968_row10_col9\" class=\"data row10 col9\" >0.1860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_e8968_level0_row11\" class=\"row_heading level0 row11\" >svm</th>\n",
       "      <td id=\"T_e8968_row11_col0\" class=\"data row11 col0\" >SVM - Linear Kernel</td>\n",
       "      <td id=\"T_e8968_row11_col1\" class=\"data row11 col1\" >0.5627</td>\n",
       "      <td id=\"T_e8968_row11_col2\" class=\"data row11 col2\" >0.0000</td>\n",
       "      <td id=\"T_e8968_row11_col3\" class=\"data row11 col3\" >0.4333</td>\n",
       "      <td id=\"T_e8968_row11_col4\" class=\"data row11 col4\" >0.1446</td>\n",
       "      <td id=\"T_e8968_row11_col5\" class=\"data row11 col5\" >0.1450</td>\n",
       "      <td id=\"T_e8968_row11_col6\" class=\"data row11 col6\" >0.0238</td>\n",
       "      <td id=\"T_e8968_row11_col7\" class=\"data row11 col7\" >0.0271</td>\n",
       "      <td id=\"T_e8968_row11_col8\" class=\"data row11 col8\" >0.0000</td>\n",
       "      <td id=\"T_e8968_row11_col9\" class=\"data row11 col9\" >0.4220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_e8968_level0_row12\" class=\"row_heading level0 row12\" >ridge</th>\n",
       "      <td id=\"T_e8968_row12_col0\" class=\"data row12 col0\" >Ridge Classifier</td>\n",
       "      <td id=\"T_e8968_row12_col1\" class=\"data row12 col1\" >0.9655</td>\n",
       "      <td id=\"T_e8968_row12_col2\" class=\"data row12 col2\" >0.0000</td>\n",
       "      <td id=\"T_e8968_row12_col3\" class=\"data row12 col3\" >0.8263</td>\n",
       "      <td id=\"T_e8968_row12_col4\" class=\"data row12 col4\" >0.9439</td>\n",
       "      <td id=\"T_e8968_row12_col5\" class=\"data row12 col5\" >0.8801</td>\n",
       "      <td id=\"T_e8968_row12_col6\" class=\"data row12 col6\" >0.8601</td>\n",
       "      <td id=\"T_e8968_row12_col7\" class=\"data row12 col7\" >0.8634</td>\n",
       "      <td id=\"T_e8968_row12_col8\" class=\"data row12 col8\" >0.0000</td>\n",
       "      <td id=\"T_e8968_row12_col9\" class=\"data row12 col9\" >0.1820</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x1a612da77c0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "clf1 = setup(silent=True,\n",
    "             data = df_train, \n",
    "             target = 'target',\n",
    "             numeric_features=numeric_feature_list,\n",
    "             log_experiment = True,\n",
    "             use_gpu=False,\n",
    "             experiment_name = 'fixing imbalance',\n",
    "             fix_imbalance = True, \n",
    "            )\n",
    "add_metric('apc', 'APC', average_precision_score, target = 'pred_proba')\n",
    "best = compare_models(sort=\"APC\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Removing outliers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_6014b th {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_6014b_row0_col0, #T_6014b_row0_col1, #T_6014b_row0_col3, #T_6014b_row0_col4, #T_6014b_row0_col5, #T_6014b_row0_col6, #T_6014b_row0_col7, #T_6014b_row1_col0, #T_6014b_row1_col1, #T_6014b_row1_col2, #T_6014b_row1_col3, #T_6014b_row1_col5, #T_6014b_row1_col6, #T_6014b_row1_col7, #T_6014b_row1_col8, #T_6014b_row2_col0, #T_6014b_row2_col2, #T_6014b_row2_col3, #T_6014b_row2_col4, #T_6014b_row2_col8, #T_6014b_row3_col0, #T_6014b_row3_col1, #T_6014b_row3_col2, #T_6014b_row3_col3, #T_6014b_row3_col4, #T_6014b_row3_col5, #T_6014b_row3_col6, #T_6014b_row3_col7, #T_6014b_row3_col8, #T_6014b_row4_col0, #T_6014b_row4_col1, #T_6014b_row4_col2, #T_6014b_row4_col3, #T_6014b_row4_col4, #T_6014b_row4_col5, #T_6014b_row4_col6, #T_6014b_row4_col7, #T_6014b_row4_col8, #T_6014b_row5_col0, #T_6014b_row5_col1, #T_6014b_row5_col2, #T_6014b_row5_col3, #T_6014b_row5_col4, #T_6014b_row5_col5, #T_6014b_row5_col6, #T_6014b_row5_col7, #T_6014b_row5_col8, #T_6014b_row6_col0, #T_6014b_row6_col1, #T_6014b_row6_col2, #T_6014b_row6_col3, #T_6014b_row6_col4, #T_6014b_row6_col5, #T_6014b_row6_col6, #T_6014b_row6_col7, #T_6014b_row6_col8, #T_6014b_row7_col0, #T_6014b_row7_col1, #T_6014b_row7_col2, #T_6014b_row7_col3, #T_6014b_row7_col4, #T_6014b_row7_col5, #T_6014b_row7_col6, #T_6014b_row7_col7, #T_6014b_row7_col8, #T_6014b_row8_col0, #T_6014b_row8_col1, #T_6014b_row8_col2, #T_6014b_row8_col4, #T_6014b_row8_col5, #T_6014b_row8_col6, #T_6014b_row8_col7, #T_6014b_row8_col8, #T_6014b_row9_col0, #T_6014b_row9_col1, #T_6014b_row9_col2, #T_6014b_row9_col3, #T_6014b_row9_col4, #T_6014b_row9_col5, #T_6014b_row9_col6, #T_6014b_row9_col7, #T_6014b_row9_col8, #T_6014b_row10_col0, #T_6014b_row10_col1, #T_6014b_row10_col2, #T_6014b_row10_col3, #T_6014b_row10_col4, #T_6014b_row10_col5, #T_6014b_row10_col6, #T_6014b_row10_col7, #T_6014b_row10_col8, #T_6014b_row11_col0, #T_6014b_row11_col1, #T_6014b_row11_col2, #T_6014b_row11_col3, #T_6014b_row11_col4, #T_6014b_row11_col5, #T_6014b_row11_col6, #T_6014b_row11_col7, #T_6014b_row11_col8, #T_6014b_row12_col0, #T_6014b_row12_col1, #T_6014b_row12_col2, #T_6014b_row12_col3, #T_6014b_row12_col4, #T_6014b_row12_col5, #T_6014b_row12_col6, #T_6014b_row12_col7, #T_6014b_row12_col8, #T_6014b_row13_col0, #T_6014b_row13_col1, #T_6014b_row13_col2, #T_6014b_row13_col3, #T_6014b_row13_col5, #T_6014b_row13_col6, #T_6014b_row13_col7, #T_6014b_row13_col8 {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_6014b_row0_col2, #T_6014b_row0_col8, #T_6014b_row1_col4, #T_6014b_row2_col1, #T_6014b_row2_col5, #T_6014b_row2_col6, #T_6014b_row2_col7, #T_6014b_row8_col3, #T_6014b_row13_col4 {\n",
       "  text-align: left;\n",
       "  background-color: yellow;\n",
       "}\n",
       "#T_6014b_row0_col9, #T_6014b_row1_col9, #T_6014b_row2_col9, #T_6014b_row3_col9, #T_6014b_row4_col9, #T_6014b_row5_col9, #T_6014b_row6_col9, #T_6014b_row7_col9, #T_6014b_row8_col9, #T_6014b_row9_col9, #T_6014b_row10_col9, #T_6014b_row12_col9, #T_6014b_row13_col9 {\n",
       "  text-align: left;\n",
       "  background-color: lightgrey;\n",
       "}\n",
       "#T_6014b_row11_col9 {\n",
       "  text-align: left;\n",
       "  background-color: yellow;\n",
       "  background-color: lightgrey;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_6014b\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_6014b_level0_col0\" class=\"col_heading level0 col0\" >Model</th>\n",
       "      <th id=\"T_6014b_level0_col1\" class=\"col_heading level0 col1\" >Accuracy</th>\n",
       "      <th id=\"T_6014b_level0_col2\" class=\"col_heading level0 col2\" >AUC</th>\n",
       "      <th id=\"T_6014b_level0_col3\" class=\"col_heading level0 col3\" >Recall</th>\n",
       "      <th id=\"T_6014b_level0_col4\" class=\"col_heading level0 col4\" >Prec.</th>\n",
       "      <th id=\"T_6014b_level0_col5\" class=\"col_heading level0 col5\" >F1</th>\n",
       "      <th id=\"T_6014b_level0_col6\" class=\"col_heading level0 col6\" >Kappa</th>\n",
       "      <th id=\"T_6014b_level0_col7\" class=\"col_heading level0 col7\" >MCC</th>\n",
       "      <th id=\"T_6014b_level0_col8\" class=\"col_heading level0 col8\" >APC</th>\n",
       "      <th id=\"T_6014b_level0_col9\" class=\"col_heading level0 col9\" >TT (Sec)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_6014b_level0_row0\" class=\"row_heading level0 row0\" >lda</th>\n",
       "      <td id=\"T_6014b_row0_col0\" class=\"data row0 col0\" >Linear Discriminant Analysis</td>\n",
       "      <td id=\"T_6014b_row0_col1\" class=\"data row0 col1\" >0.9711</td>\n",
       "      <td id=\"T_6014b_row0_col2\" class=\"data row0 col2\" >0.9775</td>\n",
       "      <td id=\"T_6014b_row0_col3\" class=\"data row0 col3\" >0.7375</td>\n",
       "      <td id=\"T_6014b_row0_col4\" class=\"data row0 col4\" >0.9917</td>\n",
       "      <td id=\"T_6014b_row0_col5\" class=\"data row0 col5\" >0.8434</td>\n",
       "      <td id=\"T_6014b_row0_col6\" class=\"data row0 col6\" >0.8280</td>\n",
       "      <td id=\"T_6014b_row0_col7\" class=\"data row0 col7\" >0.8404</td>\n",
       "      <td id=\"T_6014b_row0_col8\" class=\"data row0 col8\" >0.9311</td>\n",
       "      <td id=\"T_6014b_row0_col9\" class=\"data row0 col9\" >0.1040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6014b_level0_row1\" class=\"row_heading level0 row1\" >et</th>\n",
       "      <td id=\"T_6014b_row1_col0\" class=\"data row1 col0\" >Extra Trees Classifier</td>\n",
       "      <td id=\"T_6014b_row1_col1\" class=\"data row1 col1\" >0.9798</td>\n",
       "      <td id=\"T_6014b_row1_col2\" class=\"data row1 col2\" >0.9742</td>\n",
       "      <td id=\"T_6014b_row1_col3\" class=\"data row1 col3\" >0.8125</td>\n",
       "      <td id=\"T_6014b_row1_col4\" class=\"data row1 col4\" >1.0000</td>\n",
       "      <td id=\"T_6014b_row1_col5\" class=\"data row1 col5\" >0.8944</td>\n",
       "      <td id=\"T_6014b_row1_col6\" class=\"data row1 col6\" >0.8835</td>\n",
       "      <td id=\"T_6014b_row1_col7\" class=\"data row1 col7\" >0.8905</td>\n",
       "      <td id=\"T_6014b_row1_col8\" class=\"data row1 col8\" >0.9267</td>\n",
       "      <td id=\"T_6014b_row1_col9\" class=\"data row1 col9\" >0.5480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6014b_level0_row2\" class=\"row_heading level0 row2\" >lightgbm</th>\n",
       "      <td id=\"T_6014b_row2_col0\" class=\"data row2 col0\" >Light Gradient Boosting Machine</td>\n",
       "      <td id=\"T_6014b_row2_col1\" class=\"data row2 col1\" >0.9805</td>\n",
       "      <td id=\"T_6014b_row2_col2\" class=\"data row2 col2\" >0.9669</td>\n",
       "      <td id=\"T_6014b_row2_col3\" class=\"data row2 col3\" >0.8312</td>\n",
       "      <td id=\"T_6014b_row2_col4\" class=\"data row2 col4\" >0.9862</td>\n",
       "      <td id=\"T_6014b_row2_col5\" class=\"data row2 col5\" >0.9000</td>\n",
       "      <td id=\"T_6014b_row2_col6\" class=\"data row2 col6\" >0.8894</td>\n",
       "      <td id=\"T_6014b_row2_col7\" class=\"data row2 col7\" >0.8946</td>\n",
       "      <td id=\"T_6014b_row2_col8\" class=\"data row2 col8\" >0.9198</td>\n",
       "      <td id=\"T_6014b_row2_col9\" class=\"data row2 col9\" >1.1220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6014b_level0_row3\" class=\"row_heading level0 row3\" >gbc</th>\n",
       "      <td id=\"T_6014b_row3_col0\" class=\"data row3 col0\" >Gradient Boosting Classifier</td>\n",
       "      <td id=\"T_6014b_row3_col1\" class=\"data row3 col1\" >0.9764</td>\n",
       "      <td id=\"T_6014b_row3_col2\" class=\"data row3 col2\" >0.9624</td>\n",
       "      <td id=\"T_6014b_row3_col3\" class=\"data row3 col3\" >0.8375</td>\n",
       "      <td id=\"T_6014b_row3_col4\" class=\"data row3 col4\" >0.9430</td>\n",
       "      <td id=\"T_6014b_row3_col5\" class=\"data row3 col5\" >0.8831</td>\n",
       "      <td id=\"T_6014b_row3_col6\" class=\"data row3 col6\" >0.8702</td>\n",
       "      <td id=\"T_6014b_row3_col7\" class=\"data row3 col7\" >0.8744</td>\n",
       "      <td id=\"T_6014b_row3_col8\" class=\"data row3 col8\" >0.9187</td>\n",
       "      <td id=\"T_6014b_row3_col9\" class=\"data row3 col9\" >3.2730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6014b_level0_row4\" class=\"row_heading level0 row4\" >rf</th>\n",
       "      <td id=\"T_6014b_row4_col0\" class=\"data row4 col0\" >Random Forest Classifier</td>\n",
       "      <td id=\"T_6014b_row4_col1\" class=\"data row4 col1\" >0.9785</td>\n",
       "      <td id=\"T_6014b_row4_col2\" class=\"data row4 col2\" >0.9688</td>\n",
       "      <td id=\"T_6014b_row4_col3\" class=\"data row4 col3\" >0.8062</td>\n",
       "      <td id=\"T_6014b_row4_col4\" class=\"data row4 col4\" >0.9923</td>\n",
       "      <td id=\"T_6014b_row4_col5\" class=\"data row4 col5\" >0.8875</td>\n",
       "      <td id=\"T_6014b_row4_col6\" class=\"data row4 col6\" >0.8758</td>\n",
       "      <td id=\"T_6014b_row4_col7\" class=\"data row4 col7\" >0.8828</td>\n",
       "      <td id=\"T_6014b_row4_col8\" class=\"data row4 col8\" >0.9115</td>\n",
       "      <td id=\"T_6014b_row4_col9\" class=\"data row4 col9\" >1.2620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6014b_level0_row5\" class=\"row_heading level0 row5\" >ada</th>\n",
       "      <td id=\"T_6014b_row5_col0\" class=\"data row5 col0\" >Ada Boost Classifier</td>\n",
       "      <td id=\"T_6014b_row5_col1\" class=\"data row5 col1\" >0.9724</td>\n",
       "      <td id=\"T_6014b_row5_col2\" class=\"data row5 col2\" >0.9619</td>\n",
       "      <td id=\"T_6014b_row5_col3\" class=\"data row5 col3\" >0.8062</td>\n",
       "      <td id=\"T_6014b_row5_col4\" class=\"data row5 col4\" >0.9316</td>\n",
       "      <td id=\"T_6014b_row5_col5\" class=\"data row5 col5\" >0.8612</td>\n",
       "      <td id=\"T_6014b_row5_col6\" class=\"data row5 col6\" >0.8460</td>\n",
       "      <td id=\"T_6014b_row5_col7\" class=\"data row5 col7\" >0.8506</td>\n",
       "      <td id=\"T_6014b_row5_col8\" class=\"data row5 col8\" >0.9103</td>\n",
       "      <td id=\"T_6014b_row5_col9\" class=\"data row5 col9\" >0.9410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6014b_level0_row6\" class=\"row_heading level0 row6\" >nb</th>\n",
       "      <td id=\"T_6014b_row6_col0\" class=\"data row6 col0\" >Naive Bayes</td>\n",
       "      <td id=\"T_6014b_row6_col1\" class=\"data row6 col1\" >0.9664</td>\n",
       "      <td id=\"T_6014b_row6_col2\" class=\"data row6 col2\" >0.9684</td>\n",
       "      <td id=\"T_6014b_row6_col3\" class=\"data row6 col3\" >0.7500</td>\n",
       "      <td id=\"T_6014b_row6_col4\" class=\"data row6 col4\" >0.9266</td>\n",
       "      <td id=\"T_6014b_row6_col5\" class=\"data row6 col5\" >0.8270</td>\n",
       "      <td id=\"T_6014b_row6_col6\" class=\"data row6 col6\" >0.8086</td>\n",
       "      <td id=\"T_6014b_row6_col7\" class=\"data row6 col7\" >0.8153</td>\n",
       "      <td id=\"T_6014b_row6_col8\" class=\"data row6 col8\" >0.9019</td>\n",
       "      <td id=\"T_6014b_row6_col9\" class=\"data row6 col9\" >0.0500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6014b_level0_row7\" class=\"row_heading level0 row7\" >lr</th>\n",
       "      <td id=\"T_6014b_row7_col0\" class=\"data row7 col0\" >Logistic Regression</td>\n",
       "      <td id=\"T_6014b_row7_col1\" class=\"data row7 col1\" >0.9738</td>\n",
       "      <td id=\"T_6014b_row7_col2\" class=\"data row7 col2\" >0.9548</td>\n",
       "      <td id=\"T_6014b_row7_col3\" class=\"data row7 col3\" >0.8250</td>\n",
       "      <td id=\"T_6014b_row7_col4\" class=\"data row7 col4\" >0.9233</td>\n",
       "      <td id=\"T_6014b_row7_col5\" class=\"data row7 col5\" >0.8696</td>\n",
       "      <td id=\"T_6014b_row7_col6\" class=\"data row7 col6\" >0.8551</td>\n",
       "      <td id=\"T_6014b_row7_col7\" class=\"data row7 col7\" >0.8578</td>\n",
       "      <td id=\"T_6014b_row7_col8\" class=\"data row7 col8\" >0.9011</td>\n",
       "      <td id=\"T_6014b_row7_col9\" class=\"data row7 col9\" >0.1660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6014b_level0_row8\" class=\"row_heading level0 row8\" >qda</th>\n",
       "      <td id=\"T_6014b_row8_col0\" class=\"data row8 col0\" >Quadratic Discriminant Analysis</td>\n",
       "      <td id=\"T_6014b_row8_col1\" class=\"data row8 col1\" >0.9415</td>\n",
       "      <td id=\"T_6014b_row8_col2\" class=\"data row8 col2\" >0.9644</td>\n",
       "      <td id=\"T_6014b_row8_col3\" class=\"data row8 col3\" >0.8688</td>\n",
       "      <td id=\"T_6014b_row8_col4\" class=\"data row8 col4\" >0.6844</td>\n",
       "      <td id=\"T_6014b_row8_col5\" class=\"data row8 col5\" >0.7625</td>\n",
       "      <td id=\"T_6014b_row8_col6\" class=\"data row8 col6\" >0.7299</td>\n",
       "      <td id=\"T_6014b_row8_col7\" class=\"data row8 col7\" >0.7386</td>\n",
       "      <td id=\"T_6014b_row8_col8\" class=\"data row8 col8\" >0.8430</td>\n",
       "      <td id=\"T_6014b_row8_col9\" class=\"data row8 col9\" >0.0440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6014b_level0_row9\" class=\"row_heading level0 row9\" >dt</th>\n",
       "      <td id=\"T_6014b_row9_col0\" class=\"data row9 col0\" >Decision Tree Classifier</td>\n",
       "      <td id=\"T_6014b_row9_col1\" class=\"data row9 col1\" >0.9643</td>\n",
       "      <td id=\"T_6014b_row9_col2\" class=\"data row9 col2\" >0.9058</td>\n",
       "      <td id=\"T_6014b_row9_col3\" class=\"data row9 col3\" >0.8312</td>\n",
       "      <td id=\"T_6014b_row9_col4\" class=\"data row9 col4\" >0.8411</td>\n",
       "      <td id=\"T_6014b_row9_col5\" class=\"data row9 col5\" >0.8331</td>\n",
       "      <td id=\"T_6014b_row9_col6\" class=\"data row9 col6\" >0.8133</td>\n",
       "      <td id=\"T_6014b_row9_col7\" class=\"data row9 col7\" >0.8151</td>\n",
       "      <td id=\"T_6014b_row9_col8\" class=\"data row9 col8\" >0.7202</td>\n",
       "      <td id=\"T_6014b_row9_col9\" class=\"data row9 col9\" >0.1800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6014b_level0_row10\" class=\"row_heading level0 row10\" >knn</th>\n",
       "      <td id=\"T_6014b_row10_col0\" class=\"data row10 col0\" >K Neighbors Classifier</td>\n",
       "      <td id=\"T_6014b_row10_col1\" class=\"data row10 col1\" >0.8823</td>\n",
       "      <td id=\"T_6014b_row10_col2\" class=\"data row10 col2\" >0.5999</td>\n",
       "      <td id=\"T_6014b_row10_col3\" class=\"data row10 col3\" >0.0750</td>\n",
       "      <td id=\"T_6014b_row10_col4\" class=\"data row10 col4\" >0.3629</td>\n",
       "      <td id=\"T_6014b_row10_col5\" class=\"data row10 col5\" >0.1165</td>\n",
       "      <td id=\"T_6014b_row10_col6\" class=\"data row10 col6\" >0.0797</td>\n",
       "      <td id=\"T_6014b_row10_col7\" class=\"data row10 col7\" >0.1116</td>\n",
       "      <td id=\"T_6014b_row10_col8\" class=\"data row10 col8\" >0.1845</td>\n",
       "      <td id=\"T_6014b_row10_col9\" class=\"data row10 col9\" >0.2770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6014b_level0_row11\" class=\"row_heading level0 row11\" >dummy</th>\n",
       "      <td id=\"T_6014b_row11_col0\" class=\"data row11 col0\" >Dummy Classifier</td>\n",
       "      <td id=\"T_6014b_row11_col1\" class=\"data row11 col1\" >0.8923</td>\n",
       "      <td id=\"T_6014b_row11_col2\" class=\"data row11 col2\" >0.5000</td>\n",
       "      <td id=\"T_6014b_row11_col3\" class=\"data row11 col3\" >0.0000</td>\n",
       "      <td id=\"T_6014b_row11_col4\" class=\"data row11 col4\" >0.0000</td>\n",
       "      <td id=\"T_6014b_row11_col5\" class=\"data row11 col5\" >0.0000</td>\n",
       "      <td id=\"T_6014b_row11_col6\" class=\"data row11 col6\" >0.0000</td>\n",
       "      <td id=\"T_6014b_row11_col7\" class=\"data row11 col7\" >0.0000</td>\n",
       "      <td id=\"T_6014b_row11_col8\" class=\"data row11 col8\" >0.1077</td>\n",
       "      <td id=\"T_6014b_row11_col9\" class=\"data row11 col9\" >0.0290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6014b_level0_row12\" class=\"row_heading level0 row12\" >svm</th>\n",
       "      <td id=\"T_6014b_row12_col0\" class=\"data row12 col0\" >SVM - Linear Kernel</td>\n",
       "      <td id=\"T_6014b_row12_col1\" class=\"data row12 col1\" >0.8091</td>\n",
       "      <td id=\"T_6014b_row12_col2\" class=\"data row12 col2\" >0.0000</td>\n",
       "      <td id=\"T_6014b_row12_col3\" class=\"data row12 col3\" >0.1000</td>\n",
       "      <td id=\"T_6014b_row12_col4\" class=\"data row12 col4\" >0.0107</td>\n",
       "      <td id=\"T_6014b_row12_col5\" class=\"data row12 col5\" >0.0194</td>\n",
       "      <td id=\"T_6014b_row12_col6\" class=\"data row12 col6\" >-0.0082</td>\n",
       "      <td id=\"T_6014b_row12_col7\" class=\"data row12 col7\" >-0.0127</td>\n",
       "      <td id=\"T_6014b_row12_col8\" class=\"data row12 col8\" >0.0000</td>\n",
       "      <td id=\"T_6014b_row12_col9\" class=\"data row12 col9\" >0.0710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6014b_level0_row13\" class=\"row_heading level0 row13\" >ridge</th>\n",
       "      <td id=\"T_6014b_row13_col0\" class=\"data row13 col0\" >Ridge Classifier</td>\n",
       "      <td id=\"T_6014b_row13_col1\" class=\"data row13 col1\" >0.9704</td>\n",
       "      <td id=\"T_6014b_row13_col2\" class=\"data row13 col2\" >0.0000</td>\n",
       "      <td id=\"T_6014b_row13_col3\" class=\"data row13 col3\" >0.7250</td>\n",
       "      <td id=\"T_6014b_row13_col4\" class=\"data row13 col4\" >1.0000</td>\n",
       "      <td id=\"T_6014b_row13_col5\" class=\"data row13 col5\" >0.8372</td>\n",
       "      <td id=\"T_6014b_row13_col6\" class=\"data row13 col6\" >0.8216</td>\n",
       "      <td id=\"T_6014b_row13_col7\" class=\"data row13 col7\" >0.8363</td>\n",
       "      <td id=\"T_6014b_row13_col8\" class=\"data row13 col8\" >0.0000</td>\n",
       "      <td id=\"T_6014b_row13_col9\" class=\"data row13 col9\" >0.0380</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x1a615293c40>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "clf1 = setup(data = df_train, \n",
    "             target = 'target', \n",
    "             numeric_features=numeric_feature_list,\n",
    "             silent=True,\n",
    "             log_experiment = True,\n",
    "             use_gpu=False,\n",
    "             experiment_name = 'removing outliers',\n",
    "             remove_outliers = True\n",
    "            )\n",
    "add_metric('apc', 'APC', average_precision_score, target = 'pred_proba')\n",
    "best = compare_models(sort=\"APC\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Transformation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_6af72 th {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_6af72_row0_col0, #T_6af72_row0_col1, #T_6af72_row0_col3, #T_6af72_row0_col5, #T_6af72_row0_col6, #T_6af72_row0_col7, #T_6af72_row1_col0, #T_6af72_row1_col2, #T_6af72_row1_col3, #T_6af72_row1_col4, #T_6af72_row1_col5, #T_6af72_row1_col6, #T_6af72_row1_col7, #T_6af72_row1_col8, #T_6af72_row2_col0, #T_6af72_row2_col1, #T_6af72_row2_col2, #T_6af72_row2_col4, #T_6af72_row2_col5, #T_6af72_row2_col6, #T_6af72_row2_col7, #T_6af72_row2_col8, #T_6af72_row3_col0, #T_6af72_row3_col2, #T_6af72_row3_col3, #T_6af72_row3_col4, #T_6af72_row3_col8, #T_6af72_row4_col0, #T_6af72_row4_col1, #T_6af72_row4_col2, #T_6af72_row4_col3, #T_6af72_row4_col4, #T_6af72_row4_col5, #T_6af72_row4_col6, #T_6af72_row4_col7, #T_6af72_row4_col8, #T_6af72_row5_col0, #T_6af72_row5_col1, #T_6af72_row5_col2, #T_6af72_row5_col3, #T_6af72_row5_col4, #T_6af72_row5_col5, #T_6af72_row5_col6, #T_6af72_row5_col7, #T_6af72_row5_col8, #T_6af72_row6_col0, #T_6af72_row6_col1, #T_6af72_row6_col2, #T_6af72_row6_col3, #T_6af72_row6_col4, #T_6af72_row6_col5, #T_6af72_row6_col6, #T_6af72_row6_col7, #T_6af72_row6_col8, #T_6af72_row7_col0, #T_6af72_row7_col1, #T_6af72_row7_col2, #T_6af72_row7_col3, #T_6af72_row7_col4, #T_6af72_row7_col5, #T_6af72_row7_col6, #T_6af72_row7_col7, #T_6af72_row7_col8, #T_6af72_row8_col0, #T_6af72_row8_col1, #T_6af72_row8_col2, #T_6af72_row8_col3, #T_6af72_row8_col4, #T_6af72_row8_col5, #T_6af72_row8_col6, #T_6af72_row8_col7, #T_6af72_row8_col8, #T_6af72_row9_col0, #T_6af72_row9_col1, #T_6af72_row9_col2, #T_6af72_row9_col3, #T_6af72_row9_col4, #T_6af72_row9_col5, #T_6af72_row9_col6, #T_6af72_row9_col7, #T_6af72_row9_col8, #T_6af72_row10_col0, #T_6af72_row10_col1, #T_6af72_row10_col2, #T_6af72_row10_col3, #T_6af72_row10_col4, #T_6af72_row10_col5, #T_6af72_row10_col6, #T_6af72_row10_col7, #T_6af72_row10_col8, #T_6af72_row11_col0, #T_6af72_row11_col1, #T_6af72_row11_col2, #T_6af72_row11_col3, #T_6af72_row11_col4, #T_6af72_row11_col5, #T_6af72_row11_col6, #T_6af72_row11_col7, #T_6af72_row11_col8, #T_6af72_row12_col0, #T_6af72_row12_col1, #T_6af72_row12_col2, #T_6af72_row12_col3, #T_6af72_row12_col4, #T_6af72_row12_col5, #T_6af72_row12_col6, #T_6af72_row12_col7, #T_6af72_row12_col8, #T_6af72_row13_col0, #T_6af72_row13_col1, #T_6af72_row13_col2, #T_6af72_row13_col3, #T_6af72_row13_col4, #T_6af72_row13_col5, #T_6af72_row13_col6, #T_6af72_row13_col7, #T_6af72_row13_col8 {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_6af72_row0_col2, #T_6af72_row0_col4, #T_6af72_row0_col8, #T_6af72_row1_col1, #T_6af72_row2_col3, #T_6af72_row3_col1, #T_6af72_row3_col5, #T_6af72_row3_col6, #T_6af72_row3_col7 {\n",
       "  text-align: left;\n",
       "  background-color: yellow;\n",
       "}\n",
       "#T_6af72_row0_col9, #T_6af72_row1_col9, #T_6af72_row2_col9, #T_6af72_row3_col9, #T_6af72_row4_col9, #T_6af72_row5_col9, #T_6af72_row6_col9, #T_6af72_row7_col9, #T_6af72_row8_col9, #T_6af72_row9_col9, #T_6af72_row10_col9, #T_6af72_row12_col9, #T_6af72_row13_col9 {\n",
       "  text-align: left;\n",
       "  background-color: lightgrey;\n",
       "}\n",
       "#T_6af72_row11_col9 {\n",
       "  text-align: left;\n",
       "  background-color: yellow;\n",
       "  background-color: lightgrey;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_6af72\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_6af72_level0_col0\" class=\"col_heading level0 col0\" >Model</th>\n",
       "      <th id=\"T_6af72_level0_col1\" class=\"col_heading level0 col1\" >Accuracy</th>\n",
       "      <th id=\"T_6af72_level0_col2\" class=\"col_heading level0 col2\" >AUC</th>\n",
       "      <th id=\"T_6af72_level0_col3\" class=\"col_heading level0 col3\" >Recall</th>\n",
       "      <th id=\"T_6af72_level0_col4\" class=\"col_heading level0 col4\" >Prec.</th>\n",
       "      <th id=\"T_6af72_level0_col5\" class=\"col_heading level0 col5\" >F1</th>\n",
       "      <th id=\"T_6af72_level0_col6\" class=\"col_heading level0 col6\" >Kappa</th>\n",
       "      <th id=\"T_6af72_level0_col7\" class=\"col_heading level0 col7\" >MCC</th>\n",
       "      <th id=\"T_6af72_level0_col8\" class=\"col_heading level0 col8\" >APC</th>\n",
       "      <th id=\"T_6af72_level0_col9\" class=\"col_heading level0 col9\" >TT (Sec)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_6af72_level0_row0\" class=\"row_heading level0 row0\" >et</th>\n",
       "      <td id=\"T_6af72_row0_col0\" class=\"data row0 col0\" >Extra Trees Classifier</td>\n",
       "      <td id=\"T_6af72_row0_col1\" class=\"data row0 col1\" >0.9744</td>\n",
       "      <td id=\"T_6af72_row0_col2\" class=\"data row0 col2\" >0.9831</td>\n",
       "      <td id=\"T_6af72_row0_col3\" class=\"data row0 col3\" >0.8409</td>\n",
       "      <td id=\"T_6af72_row0_col4\" class=\"data row0 col4\" >0.9907</td>\n",
       "      <td id=\"T_6af72_row0_col5\" class=\"data row0 col5\" >0.9085</td>\n",
       "      <td id=\"T_6af72_row0_col6\" class=\"data row0 col6\" >0.8938</td>\n",
       "      <td id=\"T_6af72_row0_col7\" class=\"data row0 col7\" >0.8986</td>\n",
       "      <td id=\"T_6af72_row0_col8\" class=\"data row0 col8\" >0.9544</td>\n",
       "      <td id=\"T_6af72_row0_col9\" class=\"data row0 col9\" >1.3610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6af72_level0_row1\" class=\"row_heading level0 row1\" >lightgbm</th>\n",
       "      <td id=\"T_6af72_row1_col0\" class=\"data row1 col0\" >Light Gradient Boosting Machine</td>\n",
       "      <td id=\"T_6af72_row1_col1\" class=\"data row1 col1\" >0.9751</td>\n",
       "      <td id=\"T_6af72_row1_col2\" class=\"data row1 col2\" >0.9799</td>\n",
       "      <td id=\"T_6af72_row1_col3\" class=\"data row1 col3\" >0.8576</td>\n",
       "      <td id=\"T_6af72_row1_col4\" class=\"data row1 col4\" >0.9765</td>\n",
       "      <td id=\"T_6af72_row1_col5\" class=\"data row1 col5\" >0.9115</td>\n",
       "      <td id=\"T_6af72_row1_col6\" class=\"data row1 col6\" >0.8971</td>\n",
       "      <td id=\"T_6af72_row1_col7\" class=\"data row1 col7\" >0.9008</td>\n",
       "      <td id=\"T_6af72_row1_col8\" class=\"data row1 col8\" >0.9530</td>\n",
       "      <td id=\"T_6af72_row1_col9\" class=\"data row1 col9\" >1.4230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6af72_level0_row2\" class=\"row_heading level0 row2\" >ada</th>\n",
       "      <td id=\"T_6af72_row2_col0\" class=\"data row2 col0\" >Ada Boost Classifier</td>\n",
       "      <td id=\"T_6af72_row2_col1\" class=\"data row2 col1\" >0.9738</td>\n",
       "      <td id=\"T_6af72_row2_col2\" class=\"data row2 col2\" >0.9792</td>\n",
       "      <td id=\"T_6af72_row2_col3\" class=\"data row2 col3\" >0.8743</td>\n",
       "      <td id=\"T_6af72_row2_col4\" class=\"data row2 col4\" >0.9504</td>\n",
       "      <td id=\"T_6af72_row2_col5\" class=\"data row2 col5\" >0.9100</td>\n",
       "      <td id=\"T_6af72_row2_col6\" class=\"data row2 col6\" >0.8947</td>\n",
       "      <td id=\"T_6af72_row2_col7\" class=\"data row2 col7\" >0.8963</td>\n",
       "      <td id=\"T_6af72_row2_col8\" class=\"data row2 col8\" >0.9519</td>\n",
       "      <td id=\"T_6af72_row2_col9\" class=\"data row2 col9\" >1.5840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6af72_level0_row3\" class=\"row_heading level0 row3\" >lr</th>\n",
       "      <td id=\"T_6af72_row3_col0\" class=\"data row3 col0\" >Logistic Regression</td>\n",
       "      <td id=\"T_6af72_row3_col1\" class=\"data row3 col1\" >0.9751</td>\n",
       "      <td id=\"T_6af72_row3_col2\" class=\"data row3 col2\" >0.9789</td>\n",
       "      <td id=\"T_6af72_row3_col3\" class=\"data row3 col3\" >0.8659</td>\n",
       "      <td id=\"T_6af72_row3_col4\" class=\"data row3 col4\" >0.9684</td>\n",
       "      <td id=\"T_6af72_row3_col5\" class=\"data row3 col5\" >0.9131</td>\n",
       "      <td id=\"T_6af72_row3_col6\" class=\"data row3 col6\" >0.8986</td>\n",
       "      <td id=\"T_6af72_row3_col7\" class=\"data row3 col7\" >0.9013</td>\n",
       "      <td id=\"T_6af72_row3_col8\" class=\"data row3 col8\" >0.9518</td>\n",
       "      <td id=\"T_6af72_row3_col9\" class=\"data row3 col9\" >0.1090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6af72_level0_row4\" class=\"row_heading level0 row4\" >rf</th>\n",
       "      <td id=\"T_6af72_row4_col0\" class=\"data row4 col0\" >Random Forest Classifier</td>\n",
       "      <td id=\"T_6af72_row4_col1\" class=\"data row4 col1\" >0.9744</td>\n",
       "      <td id=\"T_6af72_row4_col2\" class=\"data row4 col2\" >0.9796</td>\n",
       "      <td id=\"T_6af72_row4_col3\" class=\"data row4 col3\" >0.8449</td>\n",
       "      <td id=\"T_6af72_row4_col4\" class=\"data row4 col4\" >0.9847</td>\n",
       "      <td id=\"T_6af72_row4_col5\" class=\"data row4 col5\" >0.9085</td>\n",
       "      <td id=\"T_6af72_row4_col6\" class=\"data row4 col6\" >0.8938</td>\n",
       "      <td id=\"T_6af72_row4_col7\" class=\"data row4 col7\" >0.8980</td>\n",
       "      <td id=\"T_6af72_row4_col8\" class=\"data row4 col8\" >0.9507</td>\n",
       "      <td id=\"T_6af72_row4_col9\" class=\"data row4 col9\" >2.8830</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6af72_level0_row5\" class=\"row_heading level0 row5\" >gbc</th>\n",
       "      <td id=\"T_6af72_row5_col0\" class=\"data row5 col0\" >Gradient Boosting Classifier</td>\n",
       "      <td id=\"T_6af72_row5_col1\" class=\"data row5 col1\" >0.9668</td>\n",
       "      <td id=\"T_6af72_row5_col2\" class=\"data row5 col2\" >0.9780</td>\n",
       "      <td id=\"T_6af72_row5_col3\" class=\"data row5 col3\" >0.8493</td>\n",
       "      <td id=\"T_6af72_row5_col4\" class=\"data row5 col4\" >0.9275</td>\n",
       "      <td id=\"T_6af72_row5_col5\" class=\"data row5 col5\" >0.8857</td>\n",
       "      <td id=\"T_6af72_row5_col6\" class=\"data row5 col6\" >0.8663</td>\n",
       "      <td id=\"T_6af72_row5_col7\" class=\"data row5 col7\" >0.8681</td>\n",
       "      <td id=\"T_6af72_row5_col8\" class=\"data row5 col8\" >0.9484</td>\n",
       "      <td id=\"T_6af72_row5_col9\" class=\"data row5 col9\" >4.6040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6af72_level0_row6\" class=\"row_heading level0 row6\" >lda</th>\n",
       "      <td id=\"T_6af72_row6_col0\" class=\"data row6 col0\" >Linear Discriminant Analysis</td>\n",
       "      <td id=\"T_6af72_row6_col1\" class=\"data row6 col1\" >0.9706</td>\n",
       "      <td id=\"T_6af72_row6_col2\" class=\"data row6 col2\" >0.9750</td>\n",
       "      <td id=\"T_6af72_row6_col3\" class=\"data row6 col3\" >0.8283</td>\n",
       "      <td id=\"T_6af72_row6_col4\" class=\"data row6 col4\" >0.9763</td>\n",
       "      <td id=\"T_6af72_row6_col5\" class=\"data row6 col5\" >0.8950</td>\n",
       "      <td id=\"T_6af72_row6_col6\" class=\"data row6 col6\" >0.8781</td>\n",
       "      <td id=\"T_6af72_row6_col7\" class=\"data row6 col7\" >0.8829</td>\n",
       "      <td id=\"T_6af72_row6_col8\" class=\"data row6 col8\" >0.9444</td>\n",
       "      <td id=\"T_6af72_row6_col9\" class=\"data row6 col9\" >0.1180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6af72_level0_row7\" class=\"row_heading level0 row7\" >knn</th>\n",
       "      <td id=\"T_6af72_row7_col0\" class=\"data row7 col0\" >K Neighbors Classifier</td>\n",
       "      <td id=\"T_6af72_row7_col1\" class=\"data row7 col1\" >0.9687</td>\n",
       "      <td id=\"T_6af72_row7_col2\" class=\"data row7 col2\" >0.9420</td>\n",
       "      <td id=\"T_6af72_row7_col3\" class=\"data row7 col3\" >0.8156</td>\n",
       "      <td id=\"T_6af72_row7_col4\" class=\"data row7 col4\" >0.9756</td>\n",
       "      <td id=\"T_6af72_row7_col5\" class=\"data row7 col5\" >0.8865</td>\n",
       "      <td id=\"T_6af72_row7_col6\" class=\"data row7 col6\" >0.8686</td>\n",
       "      <td id=\"T_6af72_row7_col7\" class=\"data row7 col7\" >0.8745</td>\n",
       "      <td id=\"T_6af72_row7_col8\" class=\"data row7 col8\" >0.8908</td>\n",
       "      <td id=\"T_6af72_row7_col9\" class=\"data row7 col9\" >0.3670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6af72_level0_row8\" class=\"row_heading level0 row8\" >qda</th>\n",
       "      <td id=\"T_6af72_row8_col0\" class=\"data row8 col0\" >Quadratic Discriminant Analysis</td>\n",
       "      <td id=\"T_6af72_row8_col1\" class=\"data row8 col1\" >0.9520</td>\n",
       "      <td id=\"T_6af72_row8_col2\" class=\"data row8 col2\" >0.9644</td>\n",
       "      <td id=\"T_6af72_row8_col3\" class=\"data row8 col3\" >0.8701</td>\n",
       "      <td id=\"T_6af72_row8_col4\" class=\"data row8 col4\" >0.8327</td>\n",
       "      <td id=\"T_6af72_row8_col5\" class=\"data row8 col5\" >0.8486</td>\n",
       "      <td id=\"T_6af72_row8_col6\" class=\"data row8 col6\" >0.8203</td>\n",
       "      <td id=\"T_6af72_row8_col7\" class=\"data row8 col7\" >0.8222</td>\n",
       "      <td id=\"T_6af72_row8_col8\" class=\"data row8 col8\" >0.8700</td>\n",
       "      <td id=\"T_6af72_row8_col9\" class=\"data row8 col9\" >0.0530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6af72_level0_row9\" class=\"row_heading level0 row9\" >nb</th>\n",
       "      <td id=\"T_6af72_row9_col0\" class=\"data row9 col0\" >Naive Bayes</td>\n",
       "      <td id=\"T_6af72_row9_col1\" class=\"data row9 col1\" >0.9489</td>\n",
       "      <td id=\"T_6af72_row9_col2\" class=\"data row9 col2\" >0.9641</td>\n",
       "      <td id=\"T_6af72_row9_col3\" class=\"data row9 col3\" >0.8324</td>\n",
       "      <td id=\"T_6af72_row9_col4\" class=\"data row9 col4\" >0.8400</td>\n",
       "      <td id=\"T_6af72_row9_col5\" class=\"data row9 col5\" >0.8335</td>\n",
       "      <td id=\"T_6af72_row9_col6\" class=\"data row9 col6\" >0.8034</td>\n",
       "      <td id=\"T_6af72_row9_col7\" class=\"data row9 col7\" >0.8052</td>\n",
       "      <td id=\"T_6af72_row9_col8\" class=\"data row9 col8\" >0.8165</td>\n",
       "      <td id=\"T_6af72_row9_col9\" class=\"data row9 col9\" >0.0510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6af72_level0_row10\" class=\"row_heading level0 row10\" >dt</th>\n",
       "      <td id=\"T_6af72_row10_col0\" class=\"data row10 col0\" >Decision Tree Classifier</td>\n",
       "      <td id=\"T_6af72_row10_col1\" class=\"data row10 col1\" >0.9470</td>\n",
       "      <td id=\"T_6af72_row10_col2\" class=\"data row10 col2\" >0.9035</td>\n",
       "      <td id=\"T_6af72_row10_col3\" class=\"data row10 col3\" >0.8409</td>\n",
       "      <td id=\"T_6af72_row10_col4\" class=\"data row10 col4\" >0.8208</td>\n",
       "      <td id=\"T_6af72_row10_col5\" class=\"data row10 col5\" >0.8296</td>\n",
       "      <td id=\"T_6af72_row10_col6\" class=\"data row10 col6\" >0.7982</td>\n",
       "      <td id=\"T_6af72_row10_col7\" class=\"data row10 col7\" >0.7991</td>\n",
       "      <td id=\"T_6af72_row10_col8\" class=\"data row10 col8\" >0.7160</td>\n",
       "      <td id=\"T_6af72_row10_col9\" class=\"data row10 col9\" >0.1440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6af72_level0_row11\" class=\"row_heading level0 row11\" >dummy</th>\n",
       "      <td id=\"T_6af72_row11_col0\" class=\"data row11 col0\" >Dummy Classifier</td>\n",
       "      <td id=\"T_6af72_row11_col1\" class=\"data row11 col1\" >0.8473</td>\n",
       "      <td id=\"T_6af72_row11_col2\" class=\"data row11 col2\" >0.5000</td>\n",
       "      <td id=\"T_6af72_row11_col3\" class=\"data row11 col3\" >0.0000</td>\n",
       "      <td id=\"T_6af72_row11_col4\" class=\"data row11 col4\" >0.0000</td>\n",
       "      <td id=\"T_6af72_row11_col5\" class=\"data row11 col5\" >0.0000</td>\n",
       "      <td id=\"T_6af72_row11_col6\" class=\"data row11 col6\" >0.0000</td>\n",
       "      <td id=\"T_6af72_row11_col7\" class=\"data row11 col7\" >0.0000</td>\n",
       "      <td id=\"T_6af72_row11_col8\" class=\"data row11 col8\" >0.1527</td>\n",
       "      <td id=\"T_6af72_row11_col9\" class=\"data row11 col9\" >0.0330</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6af72_level0_row12\" class=\"row_heading level0 row12\" >svm</th>\n",
       "      <td id=\"T_6af72_row12_col0\" class=\"data row12 col0\" >SVM - Linear Kernel</td>\n",
       "      <td id=\"T_6af72_row12_col1\" class=\"data row12 col1\" >0.9623</td>\n",
       "      <td id=\"T_6af72_row12_col2\" class=\"data row12 col2\" >0.0000</td>\n",
       "      <td id=\"T_6af72_row12_col3\" class=\"data row12 col3\" >0.8658</td>\n",
       "      <td id=\"T_6af72_row12_col4\" class=\"data row12 col4\" >0.8953</td>\n",
       "      <td id=\"T_6af72_row12_col5\" class=\"data row12 col5\" >0.8752</td>\n",
       "      <td id=\"T_6af72_row12_col6\" class=\"data row12 col6\" >0.8531</td>\n",
       "      <td id=\"T_6af72_row12_col7\" class=\"data row12 col7\" >0.8567</td>\n",
       "      <td id=\"T_6af72_row12_col8\" class=\"data row12 col8\" >0.0000</td>\n",
       "      <td id=\"T_6af72_row12_col9\" class=\"data row12 col9\" >0.0480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6af72_level0_row13\" class=\"row_heading level0 row13\" >ridge</th>\n",
       "      <td id=\"T_6af72_row13_col0\" class=\"data row13 col0\" >Ridge Classifier</td>\n",
       "      <td id=\"T_6af72_row13_col1\" class=\"data row13 col1\" >0.9681</td>\n",
       "      <td id=\"T_6af72_row13_col2\" class=\"data row13 col2\" >0.0000</td>\n",
       "      <td id=\"T_6af72_row13_col3\" class=\"data row13 col3\" >0.8074</td>\n",
       "      <td id=\"T_6af72_row13_col4\" class=\"data row13 col4\" >0.9809</td>\n",
       "      <td id=\"T_6af72_row13_col5\" class=\"data row13 col5\" >0.8840</td>\n",
       "      <td id=\"T_6af72_row13_col6\" class=\"data row13 col6\" >0.8657</td>\n",
       "      <td id=\"T_6af72_row13_col7\" class=\"data row13 col7\" >0.8723</td>\n",
       "      <td id=\"T_6af72_row13_col8\" class=\"data row13 col8\" >0.0000</td>\n",
       "      <td id=\"T_6af72_row13_col9\" class=\"data row13 col9\" >0.0480</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x1a67b130880>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "clf1 = setup(data = df_train, \n",
    "             target = 'target', \n",
    "             numeric_features=numeric_feature_list,\n",
    "             silent=True,\n",
    "             log_experiment = True,\n",
    "             use_gpu=False,\n",
    "             experiment_name = 'transformation',\n",
    "             transformation = True, \n",
    "            )\n",
    "add_metric('apc', 'APC', average_precision_score, target = 'pred_proba')\n",
    "best = compare_models(sort=\"APC\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Feature interaction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_902a6 th {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_902a6_row0_col0, #T_902a6_row0_col1, #T_902a6_row0_col3, #T_902a6_row0_col4, #T_902a6_row0_col5, #T_902a6_row0_col6, #T_902a6_row0_col7, #T_902a6_row1_col0, #T_902a6_row1_col1, #T_902a6_row1_col2, #T_902a6_row1_col3, #T_902a6_row1_col5, #T_902a6_row1_col6, #T_902a6_row1_col7, #T_902a6_row1_col8, #T_902a6_row2_col0, #T_902a6_row2_col1, #T_902a6_row2_col2, #T_902a6_row2_col3, #T_902a6_row2_col4, #T_902a6_row2_col5, #T_902a6_row2_col6, #T_902a6_row2_col7, #T_902a6_row2_col8, #T_902a6_row3_col0, #T_902a6_row3_col1, #T_902a6_row3_col2, #T_902a6_row3_col3, #T_902a6_row3_col4, #T_902a6_row3_col5, #T_902a6_row3_col6, #T_902a6_row3_col7, #T_902a6_row3_col8, #T_902a6_row4_col0, #T_902a6_row4_col2, #T_902a6_row4_col3, #T_902a6_row4_col4, #T_902a6_row4_col8, #T_902a6_row5_col0, #T_902a6_row5_col1, #T_902a6_row5_col2, #T_902a6_row5_col3, #T_902a6_row5_col4, #T_902a6_row5_col5, #T_902a6_row5_col6, #T_902a6_row5_col7, #T_902a6_row5_col8, #T_902a6_row6_col0, #T_902a6_row6_col1, #T_902a6_row6_col2, #T_902a6_row6_col3, #T_902a6_row6_col4, #T_902a6_row6_col5, #T_902a6_row6_col6, #T_902a6_row6_col7, #T_902a6_row6_col8, #T_902a6_row7_col0, #T_902a6_row7_col1, #T_902a6_row7_col2, #T_902a6_row7_col3, #T_902a6_row7_col4, #T_902a6_row7_col5, #T_902a6_row7_col6, #T_902a6_row7_col7, #T_902a6_row7_col8, #T_902a6_row8_col0, #T_902a6_row8_col1, #T_902a6_row8_col2, #T_902a6_row8_col3, #T_902a6_row8_col4, #T_902a6_row8_col5, #T_902a6_row8_col6, #T_902a6_row8_col7, #T_902a6_row8_col8, #T_902a6_row9_col0, #T_902a6_row9_col1, #T_902a6_row9_col2, #T_902a6_row9_col3, #T_902a6_row9_col4, #T_902a6_row9_col5, #T_902a6_row9_col6, #T_902a6_row9_col7, #T_902a6_row9_col8, #T_902a6_row10_col0, #T_902a6_row10_col1, #T_902a6_row10_col2, #T_902a6_row10_col3, #T_902a6_row10_col4, #T_902a6_row10_col5, #T_902a6_row10_col6, #T_902a6_row10_col7, #T_902a6_row10_col8, #T_902a6_row11_col0, #T_902a6_row11_col1, #T_902a6_row11_col2, #T_902a6_row11_col3, #T_902a6_row11_col4, #T_902a6_row11_col5, #T_902a6_row11_col6, #T_902a6_row11_col7, #T_902a6_row11_col8, #T_902a6_row12_col0, #T_902a6_row12_col1, #T_902a6_row12_col2, #T_902a6_row12_col4, #T_902a6_row12_col5, #T_902a6_row12_col6, #T_902a6_row12_col7, #T_902a6_row12_col8, #T_902a6_row13_col0, #T_902a6_row13_col1, #T_902a6_row13_col2, #T_902a6_row13_col3, #T_902a6_row13_col4, #T_902a6_row13_col5, #T_902a6_row13_col6, #T_902a6_row13_col7, #T_902a6_row13_col8 {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_902a6_row0_col2, #T_902a6_row0_col8, #T_902a6_row1_col4, #T_902a6_row4_col1, #T_902a6_row4_col5, #T_902a6_row4_col6, #T_902a6_row4_col7, #T_902a6_row12_col3 {\n",
       "  text-align: left;\n",
       "  background-color: yellow;\n",
       "}\n",
       "#T_902a6_row0_col9, #T_902a6_row1_col9, #T_902a6_row2_col9, #T_902a6_row3_col9, #T_902a6_row4_col9, #T_902a6_row5_col9, #T_902a6_row6_col9, #T_902a6_row7_col9, #T_902a6_row8_col9, #T_902a6_row9_col9, #T_902a6_row10_col9, #T_902a6_row12_col9, #T_902a6_row13_col9 {\n",
       "  text-align: left;\n",
       "  background-color: lightgrey;\n",
       "}\n",
       "#T_902a6_row11_col9 {\n",
       "  text-align: left;\n",
       "  background-color: yellow;\n",
       "  background-color: lightgrey;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_902a6\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_902a6_level0_col0\" class=\"col_heading level0 col0\" >Model</th>\n",
       "      <th id=\"T_902a6_level0_col1\" class=\"col_heading level0 col1\" >Accuracy</th>\n",
       "      <th id=\"T_902a6_level0_col2\" class=\"col_heading level0 col2\" >AUC</th>\n",
       "      <th id=\"T_902a6_level0_col3\" class=\"col_heading level0 col3\" >Recall</th>\n",
       "      <th id=\"T_902a6_level0_col4\" class=\"col_heading level0 col4\" >Prec.</th>\n",
       "      <th id=\"T_902a6_level0_col5\" class=\"col_heading level0 col5\" >F1</th>\n",
       "      <th id=\"T_902a6_level0_col6\" class=\"col_heading level0 col6\" >Kappa</th>\n",
       "      <th id=\"T_902a6_level0_col7\" class=\"col_heading level0 col7\" >MCC</th>\n",
       "      <th id=\"T_902a6_level0_col8\" class=\"col_heading level0 col8\" >APC</th>\n",
       "      <th id=\"T_902a6_level0_col9\" class=\"col_heading level0 col9\" >TT (Sec)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_902a6_level0_row0\" class=\"row_heading level0 row0\" >lightgbm</th>\n",
       "      <td id=\"T_902a6_row0_col0\" class=\"data row0 col0\" >Light Gradient Boosting Machine</td>\n",
       "      <td id=\"T_902a6_row0_col1\" class=\"data row0 col1\" >0.9802</td>\n",
       "      <td id=\"T_902a6_row0_col2\" class=\"data row0 col2\" >0.9834</td>\n",
       "      <td id=\"T_902a6_row0_col3\" class=\"data row0 col3\" >0.8799</td>\n",
       "      <td id=\"T_902a6_row0_col4\" class=\"data row0 col4\" >0.9863</td>\n",
       "      <td id=\"T_902a6_row0_col5\" class=\"data row0 col5\" >0.9283</td>\n",
       "      <td id=\"T_902a6_row0_col6\" class=\"data row0 col6\" >0.9169</td>\n",
       "      <td id=\"T_902a6_row0_col7\" class=\"data row0 col7\" >0.9201</td>\n",
       "      <td id=\"T_902a6_row0_col8\" class=\"data row0 col8\" >0.9613</td>\n",
       "      <td id=\"T_902a6_row0_col9\" class=\"data row0 col9\" >2.0800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_902a6_level0_row1\" class=\"row_heading level0 row1\" >lda</th>\n",
       "      <td id=\"T_902a6_row1_col0\" class=\"data row1 col0\" >Linear Discriminant Analysis</td>\n",
       "      <td id=\"T_902a6_row1_col1\" class=\"data row1 col1\" >0.9764</td>\n",
       "      <td id=\"T_902a6_row1_col2\" class=\"data row1 col2\" >0.9809</td>\n",
       "      <td id=\"T_902a6_row1_col3\" class=\"data row1 col3\" >0.8496</td>\n",
       "      <td id=\"T_902a6_row1_col4\" class=\"data row1 col4\" >0.9905</td>\n",
       "      <td id=\"T_902a6_row1_col5\" class=\"data row1 col5\" >0.9128</td>\n",
       "      <td id=\"T_902a6_row1_col6\" class=\"data row1 col6\" >0.8993</td>\n",
       "      <td id=\"T_902a6_row1_col7\" class=\"data row1 col7\" >0.9040</td>\n",
       "      <td id=\"T_902a6_row1_col8\" class=\"data row1 col8\" >0.9578</td>\n",
       "      <td id=\"T_902a6_row1_col9\" class=\"data row1 col9\" >0.1230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_902a6_level0_row2\" class=\"row_heading level0 row2\" >gbc</th>\n",
       "      <td id=\"T_902a6_row2_col0\" class=\"data row2 col0\" >Gradient Boosting Classifier</td>\n",
       "      <td id=\"T_902a6_row2_col1\" class=\"data row2 col1\" >0.9802</td>\n",
       "      <td id=\"T_902a6_row2_col2\" class=\"data row2 col2\" >0.9775</td>\n",
       "      <td id=\"T_902a6_row2_col3\" class=\"data row2 col3\" >0.8884</td>\n",
       "      <td id=\"T_902a6_row2_col4\" class=\"data row2 col4\" >0.9778</td>\n",
       "      <td id=\"T_902a6_row2_col5\" class=\"data row2 col5\" >0.9288</td>\n",
       "      <td id=\"T_902a6_row2_col6\" class=\"data row2 col6\" >0.9174</td>\n",
       "      <td id=\"T_902a6_row2_col7\" class=\"data row2 col7\" >0.9202</td>\n",
       "      <td id=\"T_902a6_row2_col8\" class=\"data row2 col8\" >0.9546</td>\n",
       "      <td id=\"T_902a6_row2_col9\" class=\"data row2 col9\" >9.5430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_902a6_level0_row3\" class=\"row_heading level0 row3\" >rf</th>\n",
       "      <td id=\"T_902a6_row3_col0\" class=\"data row3 col0\" >Random Forest Classifier</td>\n",
       "      <td id=\"T_902a6_row3_col1\" class=\"data row3 col1\" >0.9802</td>\n",
       "      <td id=\"T_902a6_row3_col2\" class=\"data row3 col2\" >0.9757</td>\n",
       "      <td id=\"T_902a6_row3_col3\" class=\"data row3 col3\" >0.8884</td>\n",
       "      <td id=\"T_902a6_row3_col4\" class=\"data row3 col4\" >0.9760</td>\n",
       "      <td id=\"T_902a6_row3_col5\" class=\"data row3 col5\" >0.9289</td>\n",
       "      <td id=\"T_902a6_row3_col6\" class=\"data row3 col6\" >0.9175</td>\n",
       "      <td id=\"T_902a6_row3_col7\" class=\"data row3 col7\" >0.9197</td>\n",
       "      <td id=\"T_902a6_row3_col8\" class=\"data row3 col8\" >0.9528</td>\n",
       "      <td id=\"T_902a6_row3_col9\" class=\"data row3 col9\" >2.5190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_902a6_level0_row4\" class=\"row_heading level0 row4\" >et</th>\n",
       "      <td id=\"T_902a6_row4_col0\" class=\"data row4 col0\" >Extra Trees Classifier</td>\n",
       "      <td id=\"T_902a6_row4_col1\" class=\"data row4 col1\" >0.9815</td>\n",
       "      <td id=\"T_902a6_row4_col2\" class=\"data row4 col2\" >0.9757</td>\n",
       "      <td id=\"T_902a6_row4_col3\" class=\"data row4 col3\" >0.8884</td>\n",
       "      <td id=\"T_902a6_row4_col4\" class=\"data row4 col4\" >0.9861</td>\n",
       "      <td id=\"T_902a6_row4_col5\" class=\"data row4 col5\" >0.9331</td>\n",
       "      <td id=\"T_902a6_row4_col6\" class=\"data row4 col6\" >0.9224</td>\n",
       "      <td id=\"T_902a6_row4_col7\" class=\"data row4 col7\" >0.9252</td>\n",
       "      <td id=\"T_902a6_row4_col8\" class=\"data row4 col8\" >0.9513</td>\n",
       "      <td id=\"T_902a6_row4_col9\" class=\"data row4 col9\" >0.7910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_902a6_level0_row5\" class=\"row_heading level0 row5\" >ada</th>\n",
       "      <td id=\"T_902a6_row5_col0\" class=\"data row5 col0\" >Ada Boost Classifier</td>\n",
       "      <td id=\"T_902a6_row5_col1\" class=\"data row5 col1\" >0.9764</td>\n",
       "      <td id=\"T_902a6_row5_col2\" class=\"data row5 col2\" >0.9652</td>\n",
       "      <td id=\"T_902a6_row5_col3\" class=\"data row5 col3\" >0.8839</td>\n",
       "      <td id=\"T_902a6_row5_col4\" class=\"data row5 col4\" >0.9534</td>\n",
       "      <td id=\"T_902a6_row5_col5\" class=\"data row5 col5\" >0.9142</td>\n",
       "      <td id=\"T_902a6_row5_col6\" class=\"data row5 col6\" >0.9007</td>\n",
       "      <td id=\"T_902a6_row5_col7\" class=\"data row5 col7\" >0.9035</td>\n",
       "      <td id=\"T_902a6_row5_col8\" class=\"data row5 col8\" >0.9475</td>\n",
       "      <td id=\"T_902a6_row5_col9\" class=\"data row5 col9\" >1.7810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_902a6_level0_row6\" class=\"row_heading level0 row6\" >lr</th>\n",
       "      <td id=\"T_902a6_row6_col0\" class=\"data row6 col0\" >Logistic Regression</td>\n",
       "      <td id=\"T_902a6_row6_col1\" class=\"data row6 col1\" >0.9751</td>\n",
       "      <td id=\"T_902a6_row6_col2\" class=\"data row6 col2\" >0.9697</td>\n",
       "      <td id=\"T_902a6_row6_col3\" class=\"data row6 col3\" >0.8926</td>\n",
       "      <td id=\"T_902a6_row6_col4\" class=\"data row6 col4\" >0.9418</td>\n",
       "      <td id=\"T_902a6_row6_col5\" class=\"data row6 col5\" >0.9131</td>\n",
       "      <td id=\"T_902a6_row6_col6\" class=\"data row6 col6\" >0.8986</td>\n",
       "      <td id=\"T_902a6_row6_col7\" class=\"data row6 col7\" >0.9013</td>\n",
       "      <td id=\"T_902a6_row6_col8\" class=\"data row6 col8\" >0.9393</td>\n",
       "      <td id=\"T_902a6_row6_col9\" class=\"data row6 col9\" >0.8020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_902a6_level0_row7\" class=\"row_heading level0 row7\" >knn</th>\n",
       "      <td id=\"T_902a6_row7_col0\" class=\"data row7 col0\" >K Neighbors Classifier</td>\n",
       "      <td id=\"T_902a6_row7_col1\" class=\"data row7 col1\" >0.9757</td>\n",
       "      <td id=\"T_902a6_row7_col2\" class=\"data row7 col2\" >0.9597</td>\n",
       "      <td id=\"T_902a6_row7_col3\" class=\"data row7 col3\" >0.8712</td>\n",
       "      <td id=\"T_902a6_row7_col4\" class=\"data row7 col4\" >0.9646</td>\n",
       "      <td id=\"T_902a6_row7_col5\" class=\"data row7 col5\" >0.9138</td>\n",
       "      <td id=\"T_902a6_row7_col6\" class=\"data row7 col6\" >0.8998</td>\n",
       "      <td id=\"T_902a6_row7_col7\" class=\"data row7 col7\" >0.9025</td>\n",
       "      <td id=\"T_902a6_row7_col8\" class=\"data row7 col8\" >0.9116</td>\n",
       "      <td id=\"T_902a6_row7_col9\" class=\"data row7 col9\" >0.2770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_902a6_level0_row8\" class=\"row_heading level0 row8\" >nb</th>\n",
       "      <td id=\"T_902a6_row8_col0\" class=\"data row8 col0\" >Naive Bayes</td>\n",
       "      <td id=\"T_902a6_row8_col1\" class=\"data row8 col1\" >0.9425</td>\n",
       "      <td id=\"T_902a6_row8_col2\" class=\"data row8 col2\" >0.9625</td>\n",
       "      <td id=\"T_902a6_row8_col3\" class=\"data row8 col3\" >0.6471</td>\n",
       "      <td id=\"T_902a6_row8_col4\" class=\"data row8 col4\" >0.9515</td>\n",
       "      <td id=\"T_902a6_row8_col5\" class=\"data row8 col5\" >0.7612</td>\n",
       "      <td id=\"T_902a6_row8_col6\" class=\"data row8 col6\" >0.7309</td>\n",
       "      <td id=\"T_902a6_row8_col7\" class=\"data row8 col7\" >0.7531</td>\n",
       "      <td id=\"T_902a6_row8_col8\" class=\"data row8 col8\" >0.8904</td>\n",
       "      <td id=\"T_902a6_row8_col9\" class=\"data row8 col9\" >0.0920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_902a6_level0_row9\" class=\"row_heading level0 row9\" >dt</th>\n",
       "      <td id=\"T_902a6_row9_col0\" class=\"data row9 col0\" >Decision Tree Classifier</td>\n",
       "      <td id=\"T_902a6_row9_col1\" class=\"data row9 col1\" >0.9604</td>\n",
       "      <td id=\"T_902a6_row9_col2\" class=\"data row9 col2\" >0.9307</td>\n",
       "      <td id=\"T_902a6_row9_col3\" class=\"data row9 col3\" >0.8884</td>\n",
       "      <td id=\"T_902a6_row9_col4\" class=\"data row9 col4\" >0.8571</td>\n",
       "      <td id=\"T_902a6_row9_col5\" class=\"data row9 col5\" >0.8694</td>\n",
       "      <td id=\"T_902a6_row9_col6\" class=\"data row9 col6\" >0.8462</td>\n",
       "      <td id=\"T_902a6_row9_col7\" class=\"data row9 col7\" >0.8485</td>\n",
       "      <td id=\"T_902a6_row9_col8\" class=\"data row9 col8\" >0.7773</td>\n",
       "      <td id=\"T_902a6_row9_col9\" class=\"data row9 col9\" >0.6220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_902a6_level0_row10\" class=\"row_heading level0 row10\" >qda</th>\n",
       "      <td id=\"T_902a6_row10_col0\" class=\"data row10 col0\" >Quadratic Discriminant Analysis</td>\n",
       "      <td id=\"T_902a6_row10_col1\" class=\"data row10 col1\" >0.6887</td>\n",
       "      <td id=\"T_902a6_row10_col2\" class=\"data row10 col2\" >0.7336</td>\n",
       "      <td id=\"T_902a6_row10_col3\" class=\"data row10 col3\" >0.7975</td>\n",
       "      <td id=\"T_902a6_row10_col4\" class=\"data row10 col4\" >0.3449</td>\n",
       "      <td id=\"T_902a6_row10_col5\" class=\"data row10 col5\" >0.4630</td>\n",
       "      <td id=\"T_902a6_row10_col6\" class=\"data row10 col6\" >0.3176</td>\n",
       "      <td id=\"T_902a6_row10_col7\" class=\"data row10 col7\" >0.3702</td>\n",
       "      <td id=\"T_902a6_row10_col8\" class=\"data row10 col8\" >0.2981</td>\n",
       "      <td id=\"T_902a6_row10_col9\" class=\"data row10 col9\" >0.0930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_902a6_level0_row11\" class=\"row_heading level0 row11\" >dummy</th>\n",
       "      <td id=\"T_902a6_row11_col0\" class=\"data row11 col0\" >Dummy Classifier</td>\n",
       "      <td id=\"T_902a6_row11_col1\" class=\"data row11 col1\" >0.8518</td>\n",
       "      <td id=\"T_902a6_row11_col2\" class=\"data row11 col2\" >0.5000</td>\n",
       "      <td id=\"T_902a6_row11_col3\" class=\"data row11 col3\" >0.0000</td>\n",
       "      <td id=\"T_902a6_row11_col4\" class=\"data row11 col4\" >0.0000</td>\n",
       "      <td id=\"T_902a6_row11_col5\" class=\"data row11 col5\" >0.0000</td>\n",
       "      <td id=\"T_902a6_row11_col6\" class=\"data row11 col6\" >0.0000</td>\n",
       "      <td id=\"T_902a6_row11_col7\" class=\"data row11 col7\" >0.0000</td>\n",
       "      <td id=\"T_902a6_row11_col8\" class=\"data row11 col8\" >0.1482</td>\n",
       "      <td id=\"T_902a6_row11_col9\" class=\"data row11 col9\" >0.0230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_902a6_level0_row12\" class=\"row_heading level0 row12\" >svm</th>\n",
       "      <td id=\"T_902a6_row12_col0\" class=\"data row12 col0\" >SVM - Linear Kernel</td>\n",
       "      <td id=\"T_902a6_row12_col1\" class=\"data row12 col1\" >0.9553</td>\n",
       "      <td id=\"T_902a6_row12_col2\" class=\"data row12 col2\" >0.0000</td>\n",
       "      <td id=\"T_902a6_row12_col3\" class=\"data row12 col3\" >0.8973</td>\n",
       "      <td id=\"T_902a6_row12_col4\" class=\"data row12 col4\" >0.8376</td>\n",
       "      <td id=\"T_902a6_row12_col5\" class=\"data row12 col5\" >0.8597</td>\n",
       "      <td id=\"T_902a6_row12_col6\" class=\"data row12 col6\" >0.8335</td>\n",
       "      <td id=\"T_902a6_row12_col7\" class=\"data row12 col7\" >0.8388</td>\n",
       "      <td id=\"T_902a6_row12_col8\" class=\"data row12 col8\" >0.0000</td>\n",
       "      <td id=\"T_902a6_row12_col9\" class=\"data row12 col9\" >0.0650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_902a6_level0_row13\" class=\"row_heading level0 row13\" >ridge</th>\n",
       "      <td id=\"T_902a6_row13_col0\" class=\"data row13 col0\" >Ridge Classifier</td>\n",
       "      <td id=\"T_902a6_row13_col1\" class=\"data row13 col1\" >0.9757</td>\n",
       "      <td id=\"T_902a6_row13_col2\" class=\"data row13 col2\" >0.0000</td>\n",
       "      <td id=\"T_902a6_row13_col3\" class=\"data row13 col3\" >0.8496</td>\n",
       "      <td id=\"T_902a6_row13_col4\" class=\"data row13 col4\" >0.9857</td>\n",
       "      <td id=\"T_902a6_row13_col5\" class=\"data row13 col5\" >0.9107</td>\n",
       "      <td id=\"T_902a6_row13_col6\" class=\"data row13 col6\" >0.8968</td>\n",
       "      <td id=\"T_902a6_row13_col7\" class=\"data row13 col7\" >0.9013</td>\n",
       "      <td id=\"T_902a6_row13_col8\" class=\"data row13 col8\" >0.0000</td>\n",
       "      <td id=\"T_902a6_row13_col9\" class=\"data row13 col9\" >0.0580</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x1a615442d30>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "clf1 = setup(data = df_train, \n",
    "             target = 'target',\n",
    "             numeric_features=numeric_feature_list,\n",
    "             silent=True,\n",
    "             log_experiment = True,\n",
    "             use_gpu=False,\n",
    "             experiment_name = 'feature interaction', \n",
    "             feature_interaction = True, feature_ratio = True, \n",
    "            )\n",
    "add_metric('apc', 'APC', average_precision_score, target = 'pred_proba')\n",
    "best = compare_models(sort=\"APC\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Polynomial features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_ed09b th {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_ed09b_row0_col0, #T_ed09b_row0_col1, #T_ed09b_row0_col3, #T_ed09b_row0_col4, #T_ed09b_row0_col5, #T_ed09b_row0_col6, #T_ed09b_row0_col7, #T_ed09b_row1_col0, #T_ed09b_row1_col2, #T_ed09b_row1_col3, #T_ed09b_row1_col4, #T_ed09b_row1_col8, #T_ed09b_row2_col0, #T_ed09b_row2_col1, #T_ed09b_row2_col2, #T_ed09b_row2_col3, #T_ed09b_row2_col4, #T_ed09b_row2_col5, #T_ed09b_row2_col6, #T_ed09b_row2_col7, #T_ed09b_row2_col8, #T_ed09b_row3_col0, #T_ed09b_row3_col1, #T_ed09b_row3_col2, #T_ed09b_row3_col3, #T_ed09b_row3_col5, #T_ed09b_row3_col6, #T_ed09b_row3_col7, #T_ed09b_row3_col8, #T_ed09b_row4_col0, #T_ed09b_row4_col1, #T_ed09b_row4_col2, #T_ed09b_row4_col3, #T_ed09b_row4_col4, #T_ed09b_row4_col5, #T_ed09b_row4_col6, #T_ed09b_row4_col7, #T_ed09b_row4_col8, #T_ed09b_row5_col0, #T_ed09b_row5_col1, #T_ed09b_row5_col2, #T_ed09b_row5_col3, #T_ed09b_row5_col4, #T_ed09b_row5_col5, #T_ed09b_row5_col6, #T_ed09b_row5_col7, #T_ed09b_row5_col8, #T_ed09b_row6_col0, #T_ed09b_row6_col1, #T_ed09b_row6_col2, #T_ed09b_row6_col3, #T_ed09b_row6_col4, #T_ed09b_row6_col5, #T_ed09b_row6_col6, #T_ed09b_row6_col7, #T_ed09b_row6_col8, #T_ed09b_row7_col0, #T_ed09b_row7_col1, #T_ed09b_row7_col2, #T_ed09b_row7_col3, #T_ed09b_row7_col4, #T_ed09b_row7_col5, #T_ed09b_row7_col6, #T_ed09b_row7_col7, #T_ed09b_row7_col8, #T_ed09b_row8_col0, #T_ed09b_row8_col1, #T_ed09b_row8_col2, #T_ed09b_row8_col4, #T_ed09b_row8_col5, #T_ed09b_row8_col6, #T_ed09b_row8_col7, #T_ed09b_row8_col8, #T_ed09b_row9_col0, #T_ed09b_row9_col1, #T_ed09b_row9_col2, #T_ed09b_row9_col3, #T_ed09b_row9_col4, #T_ed09b_row9_col5, #T_ed09b_row9_col6, #T_ed09b_row9_col7, #T_ed09b_row9_col8, #T_ed09b_row10_col0, #T_ed09b_row10_col1, #T_ed09b_row10_col2, #T_ed09b_row10_col3, #T_ed09b_row10_col4, #T_ed09b_row10_col5, #T_ed09b_row10_col6, #T_ed09b_row10_col7, #T_ed09b_row10_col8, #T_ed09b_row11_col0, #T_ed09b_row11_col1, #T_ed09b_row11_col2, #T_ed09b_row11_col3, #T_ed09b_row11_col4, #T_ed09b_row11_col5, #T_ed09b_row11_col6, #T_ed09b_row11_col7, #T_ed09b_row11_col8, #T_ed09b_row12_col0, #T_ed09b_row12_col1, #T_ed09b_row12_col2, #T_ed09b_row12_col3, #T_ed09b_row12_col4, #T_ed09b_row12_col5, #T_ed09b_row12_col6, #T_ed09b_row12_col7, #T_ed09b_row12_col8, #T_ed09b_row13_col0, #T_ed09b_row13_col1, #T_ed09b_row13_col2, #T_ed09b_row13_col3, #T_ed09b_row13_col5, #T_ed09b_row13_col6, #T_ed09b_row13_col7, #T_ed09b_row13_col8 {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_ed09b_row0_col2, #T_ed09b_row0_col8, #T_ed09b_row1_col1, #T_ed09b_row1_col5, #T_ed09b_row1_col6, #T_ed09b_row1_col7, #T_ed09b_row3_col4, #T_ed09b_row8_col3, #T_ed09b_row13_col4 {\n",
       "  text-align: left;\n",
       "  background-color: yellow;\n",
       "}\n",
       "#T_ed09b_row0_col9, #T_ed09b_row1_col9, #T_ed09b_row2_col9, #T_ed09b_row3_col9, #T_ed09b_row4_col9, #T_ed09b_row5_col9, #T_ed09b_row6_col9, #T_ed09b_row7_col9, #T_ed09b_row8_col9, #T_ed09b_row9_col9, #T_ed09b_row10_col9, #T_ed09b_row12_col9, #T_ed09b_row13_col9 {\n",
       "  text-align: left;\n",
       "  background-color: lightgrey;\n",
       "}\n",
       "#T_ed09b_row11_col9 {\n",
       "  text-align: left;\n",
       "  background-color: yellow;\n",
       "  background-color: lightgrey;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_ed09b\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_ed09b_level0_col0\" class=\"col_heading level0 col0\" >Model</th>\n",
       "      <th id=\"T_ed09b_level0_col1\" class=\"col_heading level0 col1\" >Accuracy</th>\n",
       "      <th id=\"T_ed09b_level0_col2\" class=\"col_heading level0 col2\" >AUC</th>\n",
       "      <th id=\"T_ed09b_level0_col3\" class=\"col_heading level0 col3\" >Recall</th>\n",
       "      <th id=\"T_ed09b_level0_col4\" class=\"col_heading level0 col4\" >Prec.</th>\n",
       "      <th id=\"T_ed09b_level0_col5\" class=\"col_heading level0 col5\" >F1</th>\n",
       "      <th id=\"T_ed09b_level0_col6\" class=\"col_heading level0 col6\" >Kappa</th>\n",
       "      <th id=\"T_ed09b_level0_col7\" class=\"col_heading level0 col7\" >MCC</th>\n",
       "      <th id=\"T_ed09b_level0_col8\" class=\"col_heading level0 col8\" >APC</th>\n",
       "      <th id=\"T_ed09b_level0_col9\" class=\"col_heading level0 col9\" >TT (Sec)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_ed09b_level0_row0\" class=\"row_heading level0 row0\" >lda</th>\n",
       "      <td id=\"T_ed09b_row0_col0\" class=\"data row0 col0\" >Linear Discriminant Analysis</td>\n",
       "      <td id=\"T_ed09b_row0_col1\" class=\"data row0 col1\" >0.9757</td>\n",
       "      <td id=\"T_ed09b_row0_col2\" class=\"data row0 col2\" >0.9814</td>\n",
       "      <td id=\"T_ed09b_row0_col3\" class=\"data row0 col3\" >0.8368</td>\n",
       "      <td id=\"T_ed09b_row0_col4\" class=\"data row0 col4\" >0.9894</td>\n",
       "      <td id=\"T_ed09b_row0_col5\" class=\"data row0 col5\" >0.9056</td>\n",
       "      <td id=\"T_ed09b_row0_col6\" class=\"data row0 col6\" >0.8918</td>\n",
       "      <td id=\"T_ed09b_row0_col7\" class=\"data row0 col7\" >0.8966</td>\n",
       "      <td id=\"T_ed09b_row0_col8\" class=\"data row0 col8\" >0.9539</td>\n",
       "      <td id=\"T_ed09b_row0_col9\" class=\"data row0 col9\" >0.1090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_ed09b_level0_row1\" class=\"row_heading level0 row1\" >lightgbm</th>\n",
       "      <td id=\"T_ed09b_row1_col0\" class=\"data row1 col0\" >Light Gradient Boosting Machine</td>\n",
       "      <td id=\"T_ed09b_row1_col1\" class=\"data row1 col1\" >0.9796</td>\n",
       "      <td id=\"T_ed09b_row1_col2\" class=\"data row1 col2\" >0.9729</td>\n",
       "      <td id=\"T_ed09b_row1_col3\" class=\"data row1 col3\" >0.8640</td>\n",
       "      <td id=\"T_ed09b_row1_col4\" class=\"data row1 col4\" >0.9902</td>\n",
       "      <td id=\"T_ed09b_row1_col5\" class=\"data row1 col5\" >0.9207</td>\n",
       "      <td id=\"T_ed09b_row1_col6\" class=\"data row1 col6\" >0.9091</td>\n",
       "      <td id=\"T_ed09b_row1_col7\" class=\"data row1 col7\" >0.9132</td>\n",
       "      <td id=\"T_ed09b_row1_col8\" class=\"data row1 col8\" >0.9462</td>\n",
       "      <td id=\"T_ed09b_row1_col9\" class=\"data row1 col9\" >1.4270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_ed09b_level0_row2\" class=\"row_heading level0 row2\" >rf</th>\n",
       "      <td id=\"T_ed09b_row2_col0\" class=\"data row2 col0\" >Random Forest Classifier</td>\n",
       "      <td id=\"T_ed09b_row2_col1\" class=\"data row2 col1\" >0.9783</td>\n",
       "      <td id=\"T_ed09b_row2_col2\" class=\"data row2 col2\" >0.9671</td>\n",
       "      <td id=\"T_ed09b_row2_col3\" class=\"data row2 col3\" >0.8595</td>\n",
       "      <td id=\"T_ed09b_row2_col4\" class=\"data row2 col4\" >0.9847</td>\n",
       "      <td id=\"T_ed09b_row2_col5\" class=\"data row2 col5\" >0.9164</td>\n",
       "      <td id=\"T_ed09b_row2_col6\" class=\"data row2 col6\" >0.9041</td>\n",
       "      <td id=\"T_ed09b_row2_col7\" class=\"data row2 col7\" >0.9077</td>\n",
       "      <td id=\"T_ed09b_row2_col8\" class=\"data row2 col8\" >0.9381</td>\n",
       "      <td id=\"T_ed09b_row2_col9\" class=\"data row2 col9\" >2.1150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_ed09b_level0_row3\" class=\"row_heading level0 row3\" >et</th>\n",
       "      <td id=\"T_ed09b_row3_col0\" class=\"data row3 col0\" >Extra Trees Classifier</td>\n",
       "      <td id=\"T_ed09b_row3_col1\" class=\"data row3 col1\" >0.9783</td>\n",
       "      <td id=\"T_ed09b_row3_col2\" class=\"data row3 col2\" >0.9695</td>\n",
       "      <td id=\"T_ed09b_row3_col3\" class=\"data row3 col3\" >0.8504</td>\n",
       "      <td id=\"T_ed09b_row3_col4\" class=\"data row3 col4\" >0.9950</td>\n",
       "      <td id=\"T_ed09b_row3_col5\" class=\"data row3 col5\" >0.9156</td>\n",
       "      <td id=\"T_ed09b_row3_col6\" class=\"data row3 col6\" >0.9033</td>\n",
       "      <td id=\"T_ed09b_row3_col7\" class=\"data row3 col7\" >0.9078</td>\n",
       "      <td id=\"T_ed09b_row3_col8\" class=\"data row3 col8\" >0.9377</td>\n",
       "      <td id=\"T_ed09b_row3_col9\" class=\"data row3 col9\" >1.5770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_ed09b_level0_row4\" class=\"row_heading level0 row4\" >ada</th>\n",
       "      <td id=\"T_ed09b_row4_col0\" class=\"data row4 col0\" >Ada Boost Classifier</td>\n",
       "      <td id=\"T_ed09b_row4_col1\" class=\"data row4 col1\" >0.9757</td>\n",
       "      <td id=\"T_ed09b_row4_col2\" class=\"data row4 col2\" >0.9656</td>\n",
       "      <td id=\"T_ed09b_row4_col3\" class=\"data row4 col3\" >0.8731</td>\n",
       "      <td id=\"T_ed09b_row4_col4\" class=\"data row4 col4\" >0.9535</td>\n",
       "      <td id=\"T_ed09b_row4_col5\" class=\"data row4 col5\" >0.9094</td>\n",
       "      <td id=\"T_ed09b_row4_col6\" class=\"data row4 col6\" >0.8955</td>\n",
       "      <td id=\"T_ed09b_row4_col7\" class=\"data row4 col7\" >0.8980</td>\n",
       "      <td id=\"T_ed09b_row4_col8\" class=\"data row4 col8\" >0.9356</td>\n",
       "      <td id=\"T_ed09b_row4_col9\" class=\"data row4 col9\" >1.2310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_ed09b_level0_row5\" class=\"row_heading level0 row5\" >lr</th>\n",
       "      <td id=\"T_ed09b_row5_col0\" class=\"data row5 col0\" >Logistic Regression</td>\n",
       "      <td id=\"T_ed09b_row5_col1\" class=\"data row5 col1\" >0.9713</td>\n",
       "      <td id=\"T_ed09b_row5_col2\" class=\"data row5 col2\" >0.9693</td>\n",
       "      <td id=\"T_ed09b_row5_col3\" class=\"data row5 col3\" >0.8640</td>\n",
       "      <td id=\"T_ed09b_row5_col4\" class=\"data row5 col4\" >0.9314</td>\n",
       "      <td id=\"T_ed09b_row5_col5\" class=\"data row5 col5\" >0.8945</td>\n",
       "      <td id=\"T_ed09b_row5_col6\" class=\"data row5 col6\" >0.8779</td>\n",
       "      <td id=\"T_ed09b_row5_col7\" class=\"data row5 col7\" >0.8800</td>\n",
       "      <td id=\"T_ed09b_row5_col8\" class=\"data row5 col8\" >0.9331</td>\n",
       "      <td id=\"T_ed09b_row5_col9\" class=\"data row5 col9\" >0.8020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_ed09b_level0_row6\" class=\"row_heading level0 row6\" >gbc</th>\n",
       "      <td id=\"T_ed09b_row6_col0\" class=\"data row6 col0\" >Gradient Boosting Classifier</td>\n",
       "      <td id=\"T_ed09b_row6_col1\" class=\"data row6 col1\" >0.9706</td>\n",
       "      <td id=\"T_ed09b_row6_col2\" class=\"data row6 col2\" >0.9658</td>\n",
       "      <td id=\"T_ed09b_row6_col3\" class=\"data row6 col3\" >0.8595</td>\n",
       "      <td id=\"T_ed09b_row6_col4\" class=\"data row6 col4\" >0.9262</td>\n",
       "      <td id=\"T_ed09b_row6_col5\" class=\"data row6 col5\" >0.8911</td>\n",
       "      <td id=\"T_ed09b_row6_col6\" class=\"data row6 col6\" >0.8741</td>\n",
       "      <td id=\"T_ed09b_row6_col7\" class=\"data row6 col7\" >0.8753</td>\n",
       "      <td id=\"T_ed09b_row6_col8\" class=\"data row6 col8\" >0.9272</td>\n",
       "      <td id=\"T_ed09b_row6_col9\" class=\"data row6 col9\" >3.4730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_ed09b_level0_row7\" class=\"row_heading level0 row7\" >nb</th>\n",
       "      <td id=\"T_ed09b_row7_col0\" class=\"data row7 col0\" >Naive Bayes</td>\n",
       "      <td id=\"T_ed09b_row7_col1\" class=\"data row7 col1\" >0.9693</td>\n",
       "      <td id=\"T_ed09b_row7_col2\" class=\"data row7 col2\" >0.9700</td>\n",
       "      <td id=\"T_ed09b_row7_col3\" class=\"data row7 col3\" >0.8458</td>\n",
       "      <td id=\"T_ed09b_row7_col4\" class=\"data row7 col4\" >0.9364</td>\n",
       "      <td id=\"T_ed09b_row7_col5\" class=\"data row7 col5\" >0.8856</td>\n",
       "      <td id=\"T_ed09b_row7_col6\" class=\"data row7 col6\" >0.8680</td>\n",
       "      <td id=\"T_ed09b_row7_col7\" class=\"data row7 col7\" >0.8715</td>\n",
       "      <td id=\"T_ed09b_row7_col8\" class=\"data row7 col8\" >0.8988</td>\n",
       "      <td id=\"T_ed09b_row7_col9\" class=\"data row7 col9\" >0.0380</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_ed09b_level0_row8\" class=\"row_heading level0 row8\" >qda</th>\n",
       "      <td id=\"T_ed09b_row8_col0\" class=\"data row8 col0\" >Quadratic Discriminant Analysis</td>\n",
       "      <td id=\"T_ed09b_row8_col1\" class=\"data row8 col1\" >0.9598</td>\n",
       "      <td id=\"T_ed09b_row8_col2\" class=\"data row8 col2\" >0.9675</td>\n",
       "      <td id=\"T_ed09b_row8_col3\" class=\"data row8 col3\" >0.8868</td>\n",
       "      <td id=\"T_ed09b_row8_col4\" class=\"data row8 col4\" >0.8464</td>\n",
       "      <td id=\"T_ed09b_row8_col5\" class=\"data row8 col5\" >0.8630</td>\n",
       "      <td id=\"T_ed09b_row8_col6\" class=\"data row8 col6\" >0.8396</td>\n",
       "      <td id=\"T_ed09b_row8_col7\" class=\"data row8 col7\" >0.8420</td>\n",
       "      <td id=\"T_ed09b_row8_col8\" class=\"data row8 col8\" >0.8740</td>\n",
       "      <td id=\"T_ed09b_row8_col9\" class=\"data row8 col9\" >0.0630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_ed09b_level0_row9\" class=\"row_heading level0 row9\" >dt</th>\n",
       "      <td id=\"T_ed09b_row9_col0\" class=\"data row9 col0\" >Decision Tree Classifier</td>\n",
       "      <td id=\"T_ed09b_row9_col1\" class=\"data row9 col1\" >0.9508</td>\n",
       "      <td id=\"T_ed09b_row9_col2\" class=\"data row9 col2\" >0.9050</td>\n",
       "      <td id=\"T_ed09b_row9_col3\" class=\"data row9 col3\" >0.8413</td>\n",
       "      <td id=\"T_ed09b_row9_col4\" class=\"data row9 col4\" >0.8206</td>\n",
       "      <td id=\"T_ed09b_row9_col5\" class=\"data row9 col5\" >0.8260</td>\n",
       "      <td id=\"T_ed09b_row9_col6\" class=\"data row9 col6\" >0.7975</td>\n",
       "      <td id=\"T_ed09b_row9_col7\" class=\"data row9 col7\" >0.8007</td>\n",
       "      <td id=\"T_ed09b_row9_col8\" class=\"data row9 col8\" >0.7108</td>\n",
       "      <td id=\"T_ed09b_row9_col9\" class=\"data row9 col9\" >0.1610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_ed09b_level0_row10\" class=\"row_heading level0 row10\" >knn</th>\n",
       "      <td id=\"T_ed09b_row10_col0\" class=\"data row10 col0\" >K Neighbors Classifier</td>\n",
       "      <td id=\"T_ed09b_row10_col1\" class=\"data row10 col1\" >0.8594</td>\n",
       "      <td id=\"T_ed09b_row10_col2\" class=\"data row10 col2\" >0.6932</td>\n",
       "      <td id=\"T_ed09b_row10_col3\" class=\"data row10 col3\" >0.2077</td>\n",
       "      <td id=\"T_ed09b_row10_col4\" class=\"data row10 col4\" >0.4878</td>\n",
       "      <td id=\"T_ed09b_row10_col5\" class=\"data row10 col5\" >0.2873</td>\n",
       "      <td id=\"T_ed09b_row10_col6\" class=\"data row10 col6\" >0.2259</td>\n",
       "      <td id=\"T_ed09b_row10_col7\" class=\"data row10 col7\" >0.2520</td>\n",
       "      <td id=\"T_ed09b_row10_col8\" class=\"data row10 col8\" >0.3389</td>\n",
       "      <td id=\"T_ed09b_row10_col9\" class=\"data row10 col9\" >0.0780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_ed09b_level0_row11\" class=\"row_heading level0 row11\" >dummy</th>\n",
       "      <td id=\"T_ed09b_row11_col0\" class=\"data row11 col0\" >Dummy Classifier</td>\n",
       "      <td id=\"T_ed09b_row11_col1\" class=\"data row11 col1\" >0.8588</td>\n",
       "      <td id=\"T_ed09b_row11_col2\" class=\"data row11 col2\" >0.5000</td>\n",
       "      <td id=\"T_ed09b_row11_col3\" class=\"data row11 col3\" >0.0000</td>\n",
       "      <td id=\"T_ed09b_row11_col4\" class=\"data row11 col4\" >0.0000</td>\n",
       "      <td id=\"T_ed09b_row11_col5\" class=\"data row11 col5\" >0.0000</td>\n",
       "      <td id=\"T_ed09b_row11_col6\" class=\"data row11 col6\" >0.0000</td>\n",
       "      <td id=\"T_ed09b_row11_col7\" class=\"data row11 col7\" >0.0000</td>\n",
       "      <td id=\"T_ed09b_row11_col8\" class=\"data row11 col8\" >0.1412</td>\n",
       "      <td id=\"T_ed09b_row11_col9\" class=\"data row11 col9\" >0.0240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_ed09b_level0_row12\" class=\"row_heading level0 row12\" >svm</th>\n",
       "      <td id=\"T_ed09b_row12_col0\" class=\"data row12 col0\" >SVM - Linear Kernel</td>\n",
       "      <td id=\"T_ed09b_row12_col1\" class=\"data row12 col1\" >0.6440</td>\n",
       "      <td id=\"T_ed09b_row12_col2\" class=\"data row12 col2\" >0.0000</td>\n",
       "      <td id=\"T_ed09b_row12_col3\" class=\"data row12 col3\" >0.3500</td>\n",
       "      <td id=\"T_ed09b_row12_col4\" class=\"data row12 col4\" >0.3190</td>\n",
       "      <td id=\"T_ed09b_row12_col5\" class=\"data row12 col5\" >0.1557</td>\n",
       "      <td id=\"T_ed09b_row12_col6\" class=\"data row12 col6\" >0.0628</td>\n",
       "      <td id=\"T_ed09b_row12_col7\" class=\"data row12 col7\" >0.0894</td>\n",
       "      <td id=\"T_ed09b_row12_col8\" class=\"data row12 col8\" >0.0000</td>\n",
       "      <td id=\"T_ed09b_row12_col9\" class=\"data row12 col9\" >0.0930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_ed09b_level0_row13\" class=\"row_heading level0 row13\" >ridge</th>\n",
       "      <td id=\"T_ed09b_row13_col0\" class=\"data row13 col0\" >Ridge Classifier</td>\n",
       "      <td id=\"T_ed09b_row13_col1\" class=\"data row13 col1\" >0.9757</td>\n",
       "      <td id=\"T_ed09b_row13_col2\" class=\"data row13 col2\" >0.0000</td>\n",
       "      <td id=\"T_ed09b_row13_col3\" class=\"data row13 col3\" >0.8322</td>\n",
       "      <td id=\"T_ed09b_row13_col4\" class=\"data row13 col4\" >0.9950</td>\n",
       "      <td id=\"T_ed09b_row13_col5\" class=\"data row13 col5\" >0.9051</td>\n",
       "      <td id=\"T_ed09b_row13_col6\" class=\"data row13 col6\" >0.8914</td>\n",
       "      <td id=\"T_ed09b_row13_col7\" class=\"data row13 col7\" >0.8968</td>\n",
       "      <td id=\"T_ed09b_row13_col8\" class=\"data row13 col8\" >0.0000</td>\n",
       "      <td id=\"T_ed09b_row13_col9\" class=\"data row13 col9\" >0.0390</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x1a615bb01c0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "clf1 = setup(data = df_train, \n",
    "             target = 'target', \n",
    "             numeric_features=numeric_feature_list,\n",
    "             silent=True,\n",
    "             log_experiment = True,\n",
    "             use_gpu=False,\n",
    "             experiment_name = 'polynomial features',\n",
    "             polynomial_features = True,\n",
    "            )\n",
    "add_metric('apc', 'APC', average_precision_score, target = 'pred_proba')\n",
    "best = compare_models(sort=\"APC\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Feature selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_66779 th {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_66779_row0_col0, #T_66779_row0_col1, #T_66779_row0_col3, #T_66779_row0_col4, #T_66779_row0_col5, #T_66779_row0_col6, #T_66779_row0_col7, #T_66779_row1_col0, #T_66779_row1_col1, #T_66779_row1_col2, #T_66779_row1_col3, #T_66779_row1_col4, #T_66779_row1_col5, #T_66779_row1_col6, #T_66779_row1_col7, #T_66779_row1_col8, #T_66779_row2_col0, #T_66779_row2_col2, #T_66779_row2_col3, #T_66779_row2_col8, #T_66779_row3_col0, #T_66779_row3_col1, #T_66779_row3_col2, #T_66779_row3_col3, #T_66779_row3_col4, #T_66779_row3_col5, #T_66779_row3_col6, #T_66779_row3_col7, #T_66779_row3_col8, #T_66779_row4_col0, #T_66779_row4_col1, #T_66779_row4_col2, #T_66779_row4_col3, #T_66779_row4_col4, #T_66779_row4_col5, #T_66779_row4_col6, #T_66779_row4_col7, #T_66779_row4_col8, #T_66779_row5_col0, #T_66779_row5_col1, #T_66779_row5_col2, #T_66779_row5_col3, #T_66779_row5_col4, #T_66779_row5_col5, #T_66779_row5_col6, #T_66779_row5_col7, #T_66779_row5_col8, #T_66779_row6_col0, #T_66779_row6_col1, #T_66779_row6_col2, #T_66779_row6_col3, #T_66779_row6_col4, #T_66779_row6_col5, #T_66779_row6_col6, #T_66779_row6_col7, #T_66779_row6_col8, #T_66779_row7_col0, #T_66779_row7_col1, #T_66779_row7_col2, #T_66779_row7_col3, #T_66779_row7_col4, #T_66779_row7_col5, #T_66779_row7_col6, #T_66779_row7_col7, #T_66779_row7_col8, #T_66779_row8_col0, #T_66779_row8_col1, #T_66779_row8_col2, #T_66779_row8_col4, #T_66779_row8_col5, #T_66779_row8_col6, #T_66779_row8_col7, #T_66779_row8_col8, #T_66779_row9_col0, #T_66779_row9_col1, #T_66779_row9_col2, #T_66779_row9_col3, #T_66779_row9_col4, #T_66779_row9_col5, #T_66779_row9_col6, #T_66779_row9_col7, #T_66779_row9_col8, #T_66779_row10_col0, #T_66779_row10_col1, #T_66779_row10_col2, #T_66779_row10_col3, #T_66779_row10_col4, #T_66779_row10_col5, #T_66779_row10_col6, #T_66779_row10_col7, #T_66779_row10_col8, #T_66779_row11_col0, #T_66779_row11_col1, #T_66779_row11_col2, #T_66779_row11_col3, #T_66779_row11_col4, #T_66779_row11_col5, #T_66779_row11_col6, #T_66779_row11_col7, #T_66779_row11_col8, #T_66779_row12_col0, #T_66779_row12_col1, #T_66779_row12_col2, #T_66779_row12_col3, #T_66779_row12_col4, #T_66779_row12_col5, #T_66779_row12_col6, #T_66779_row12_col7, #T_66779_row12_col8, #T_66779_row13_col0, #T_66779_row13_col1, #T_66779_row13_col2, #T_66779_row13_col3, #T_66779_row13_col5, #T_66779_row13_col6, #T_66779_row13_col7, #T_66779_row13_col8 {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_66779_row0_col2, #T_66779_row0_col8, #T_66779_row2_col1, #T_66779_row2_col4, #T_66779_row2_col5, #T_66779_row2_col6, #T_66779_row2_col7, #T_66779_row8_col3, #T_66779_row13_col4 {\n",
       "  text-align: left;\n",
       "  background-color: yellow;\n",
       "}\n",
       "#T_66779_row0_col9, #T_66779_row1_col9, #T_66779_row2_col9, #T_66779_row3_col9, #T_66779_row4_col9, #T_66779_row5_col9, #T_66779_row6_col9, #T_66779_row7_col9, #T_66779_row8_col9, #T_66779_row9_col9, #T_66779_row10_col9, #T_66779_row12_col9, #T_66779_row13_col9 {\n",
       "  text-align: left;\n",
       "  background-color: lightgrey;\n",
       "}\n",
       "#T_66779_row11_col9 {\n",
       "  text-align: left;\n",
       "  background-color: yellow;\n",
       "  background-color: lightgrey;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_66779\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_66779_level0_col0\" class=\"col_heading level0 col0\" >Model</th>\n",
       "      <th id=\"T_66779_level0_col1\" class=\"col_heading level0 col1\" >Accuracy</th>\n",
       "      <th id=\"T_66779_level0_col2\" class=\"col_heading level0 col2\" >AUC</th>\n",
       "      <th id=\"T_66779_level0_col3\" class=\"col_heading level0 col3\" >Recall</th>\n",
       "      <th id=\"T_66779_level0_col4\" class=\"col_heading level0 col4\" >Prec.</th>\n",
       "      <th id=\"T_66779_level0_col5\" class=\"col_heading level0 col5\" >F1</th>\n",
       "      <th id=\"T_66779_level0_col6\" class=\"col_heading level0 col6\" >Kappa</th>\n",
       "      <th id=\"T_66779_level0_col7\" class=\"col_heading level0 col7\" >MCC</th>\n",
       "      <th id=\"T_66779_level0_col8\" class=\"col_heading level0 col8\" >APC</th>\n",
       "      <th id=\"T_66779_level0_col9\" class=\"col_heading level0 col9\" >TT (Sec)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_66779_level0_row0\" class=\"row_heading level0 row0\" >lightgbm</th>\n",
       "      <td id=\"T_66779_row0_col0\" class=\"data row0 col0\" >Light Gradient Boosting Machine</td>\n",
       "      <td id=\"T_66779_row0_col1\" class=\"data row0 col1\" >0.9770</td>\n",
       "      <td id=\"T_66779_row0_col2\" class=\"data row0 col2\" >0.9780</td>\n",
       "      <td id=\"T_66779_row0_col3\" class=\"data row0 col3\" >0.8682</td>\n",
       "      <td id=\"T_66779_row0_col4\" class=\"data row0 col4\" >0.9813</td>\n",
       "      <td id=\"T_66779_row0_col5\" class=\"data row0 col5\" >0.9205</td>\n",
       "      <td id=\"T_66779_row0_col6\" class=\"data row0 col6\" >0.9072</td>\n",
       "      <td id=\"T_66779_row0_col7\" class=\"data row0 col7\" >0.9100</td>\n",
       "      <td id=\"T_66779_row0_col8\" class=\"data row0 col8\" >0.9538</td>\n",
       "      <td id=\"T_66779_row0_col9\" class=\"data row0 col9\" >0.8140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_66779_level0_row1\" class=\"row_heading level0 row1\" >gbc</th>\n",
       "      <td id=\"T_66779_row1_col0\" class=\"data row1 col0\" >Gradient Boosting Classifier</td>\n",
       "      <td id=\"T_66779_row1_col1\" class=\"data row1 col1\" >0.9713</td>\n",
       "      <td id=\"T_66779_row1_col2\" class=\"data row1 col2\" >0.9763</td>\n",
       "      <td id=\"T_66779_row1_col3\" class=\"data row1 col3\" >0.8680</td>\n",
       "      <td id=\"T_66779_row1_col4\" class=\"data row1 col4\" >0.9426</td>\n",
       "      <td id=\"T_66779_row1_col5\" class=\"data row1 col5\" >0.9030</td>\n",
       "      <td id=\"T_66779_row1_col6\" class=\"data row1 col6\" >0.8861</td>\n",
       "      <td id=\"T_66779_row1_col7\" class=\"data row1 col7\" >0.8877</td>\n",
       "      <td id=\"T_66779_row1_col8\" class=\"data row1 col8\" >0.9514</td>\n",
       "      <td id=\"T_66779_row1_col9\" class=\"data row1 col9\" >2.1750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_66779_level0_row2\" class=\"row_heading level0 row2\" >et</th>\n",
       "      <td id=\"T_66779_row2_col0\" class=\"data row2 col0\" >Extra Trees Classifier</td>\n",
       "      <td id=\"T_66779_row2_col1\" class=\"data row2 col1\" >0.9783</td>\n",
       "      <td id=\"T_66779_row2_col2\" class=\"data row2 col2\" >0.9743</td>\n",
       "      <td id=\"T_66779_row2_col3\" class=\"data row2 col3\" >0.8598</td>\n",
       "      <td id=\"T_66779_row2_col4\" class=\"data row2 col4\" >1.0000</td>\n",
       "      <td id=\"T_66779_row2_col5\" class=\"data row2 col5\" >0.9234</td>\n",
       "      <td id=\"T_66779_row2_col6\" class=\"data row2 col6\" >0.9108</td>\n",
       "      <td id=\"T_66779_row2_col7\" class=\"data row2 col7\" >0.9152</td>\n",
       "      <td id=\"T_66779_row2_col8\" class=\"data row2 col8\" >0.9511</td>\n",
       "      <td id=\"T_66779_row2_col9\" class=\"data row2 col9\" >1.0750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_66779_level0_row3\" class=\"row_heading level0 row3\" >rf</th>\n",
       "      <td id=\"T_66779_row3_col0\" class=\"data row3 col0\" >Random Forest Classifier</td>\n",
       "      <td id=\"T_66779_row3_col1\" class=\"data row3 col1\" >0.9757</td>\n",
       "      <td id=\"T_66779_row3_col2\" class=\"data row3 col2\" >0.9738</td>\n",
       "      <td id=\"T_66779_row3_col3\" class=\"data row3 col3\" >0.8597</td>\n",
       "      <td id=\"T_66779_row3_col4\" class=\"data row3 col4\" >0.9810</td>\n",
       "      <td id=\"T_66779_row3_col5\" class=\"data row3 col5\" >0.9157</td>\n",
       "      <td id=\"T_66779_row3_col6\" class=\"data row3 col6\" >0.9016</td>\n",
       "      <td id=\"T_66779_row3_col7\" class=\"data row3 col7\" >0.9047</td>\n",
       "      <td id=\"T_66779_row3_col8\" class=\"data row3 col8\" >0.9495</td>\n",
       "      <td id=\"T_66779_row3_col9\" class=\"data row3 col9\" >2.2420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_66779_level0_row4\" class=\"row_heading level0 row4\" >lda</th>\n",
       "      <td id=\"T_66779_row4_col0\" class=\"data row4 col0\" >Linear Discriminant Analysis</td>\n",
       "      <td id=\"T_66779_row4_col1\" class=\"data row4 col1\" >0.9572</td>\n",
       "      <td id=\"T_66779_row4_col2\" class=\"data row4 col2\" >0.9757</td>\n",
       "      <td id=\"T_66779_row4_col3\" class=\"data row4 col3\" >0.7280</td>\n",
       "      <td id=\"T_66779_row4_col4\" class=\"data row4 col4\" >0.9933</td>\n",
       "      <td id=\"T_66779_row4_col5\" class=\"data row4 col5\" >0.8380</td>\n",
       "      <td id=\"T_66779_row4_col6\" class=\"data row4 col6\" >0.8143</td>\n",
       "      <td id=\"T_66779_row4_col7\" class=\"data row4 col7\" >0.8286</td>\n",
       "      <td id=\"T_66779_row4_col8\" class=\"data row4 col8\" >0.9468</td>\n",
       "      <td id=\"T_66779_row4_col9\" class=\"data row4 col9\" >0.0500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_66779_level0_row5\" class=\"row_heading level0 row5\" >lr</th>\n",
       "      <td id=\"T_66779_row5_col0\" class=\"data row5 col0\" >Logistic Regression</td>\n",
       "      <td id=\"T_66779_row5_col1\" class=\"data row5 col1\" >0.9706</td>\n",
       "      <td id=\"T_66779_row5_col2\" class=\"data row5 col2\" >0.9664</td>\n",
       "      <td id=\"T_66779_row5_col3\" class=\"data row5 col3\" >0.8763</td>\n",
       "      <td id=\"T_66779_row5_col4\" class=\"data row5 col4\" >0.9300</td>\n",
       "      <td id=\"T_66779_row5_col5\" class=\"data row5 col5\" >0.9019</td>\n",
       "      <td id=\"T_66779_row5_col6\" class=\"data row5 col6\" >0.8847</td>\n",
       "      <td id=\"T_66779_row5_col7\" class=\"data row5 col7\" >0.8855</td>\n",
       "      <td id=\"T_66779_row5_col8\" class=\"data row5 col8\" >0.9361</td>\n",
       "      <td id=\"T_66779_row5_col9\" class=\"data row5 col9\" >0.2650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_66779_level0_row6\" class=\"row_heading level0 row6\" >ada</th>\n",
       "      <td id=\"T_66779_row6_col0\" class=\"data row6 col0\" >Ada Boost Classifier</td>\n",
       "      <td id=\"T_66779_row6_col1\" class=\"data row6 col1\" >0.9732</td>\n",
       "      <td id=\"T_66779_row6_col2\" class=\"data row6 col2\" >0.9626</td>\n",
       "      <td id=\"T_66779_row6_col3\" class=\"data row6 col3\" >0.8680</td>\n",
       "      <td id=\"T_66779_row6_col4\" class=\"data row6 col4\" >0.9568</td>\n",
       "      <td id=\"T_66779_row6_col5\" class=\"data row6 col5\" >0.9091</td>\n",
       "      <td id=\"T_66779_row6_col6\" class=\"data row6 col6\" >0.8935</td>\n",
       "      <td id=\"T_66779_row6_col7\" class=\"data row6 col7\" >0.8956</td>\n",
       "      <td id=\"T_66779_row6_col8\" class=\"data row6 col8\" >0.9330</td>\n",
       "      <td id=\"T_66779_row6_col9\" class=\"data row6 col9\" >1.5620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_66779_level0_row7\" class=\"row_heading level0 row7\" >nb</th>\n",
       "      <td id=\"T_66779_row7_col0\" class=\"data row7 col0\" >Naive Bayes</td>\n",
       "      <td id=\"T_66779_row7_col1\" class=\"data row7 col1\" >0.9547</td>\n",
       "      <td id=\"T_66779_row7_col2\" class=\"data row7 col2\" >0.9723</td>\n",
       "      <td id=\"T_66779_row7_col3\" class=\"data row7 col3\" >0.7527</td>\n",
       "      <td id=\"T_66779_row7_col4\" class=\"data row7 col4\" >0.9399</td>\n",
       "      <td id=\"T_66779_row7_col5\" class=\"data row7 col5\" >0.8345</td>\n",
       "      <td id=\"T_66779_row7_col6\" class=\"data row7 col6\" >0.8088</td>\n",
       "      <td id=\"T_66779_row7_col7\" class=\"data row7 col7\" >0.8163</td>\n",
       "      <td id=\"T_66779_row7_col8\" class=\"data row7 col8\" >0.9154</td>\n",
       "      <td id=\"T_66779_row7_col9\" class=\"data row7 col9\" >0.0540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_66779_level0_row8\" class=\"row_heading level0 row8\" >qda</th>\n",
       "      <td id=\"T_66779_row8_col0\" class=\"data row8 col0\" >Quadratic Discriminant Analysis</td>\n",
       "      <td id=\"T_66779_row8_col1\" class=\"data row8 col1\" >0.9495</td>\n",
       "      <td id=\"T_66779_row8_col2\" class=\"data row8 col2\" >0.9681</td>\n",
       "      <td id=\"T_66779_row8_col3\" class=\"data row8 col3\" >0.8927</td>\n",
       "      <td id=\"T_66779_row8_col4\" class=\"data row8 col4\" >0.8100</td>\n",
       "      <td id=\"T_66779_row8_col5\" class=\"data row8 col5\" >0.8475</td>\n",
       "      <td id=\"T_66779_row8_col6\" class=\"data row8 col6\" >0.8175</td>\n",
       "      <td id=\"T_66779_row8_col7\" class=\"data row8 col7\" >0.8202</td>\n",
       "      <td id=\"T_66779_row8_col8\" class=\"data row8 col8\" >0.8831</td>\n",
       "      <td id=\"T_66779_row8_col9\" class=\"data row8 col9\" >0.0970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_66779_level0_row9\" class=\"row_heading level0 row9\" >dt</th>\n",
       "      <td id=\"T_66779_row9_col0\" class=\"data row9 col0\" >Decision Tree Classifier</td>\n",
       "      <td id=\"T_66779_row9_col1\" class=\"data row9 col1\" >0.9591</td>\n",
       "      <td id=\"T_66779_row9_col2\" class=\"data row9 col2\" >0.9218</td>\n",
       "      <td id=\"T_66779_row9_col3\" class=\"data row9 col3\" >0.8678</td>\n",
       "      <td id=\"T_66779_row9_col4\" class=\"data row9 col4\" >0.8756</td>\n",
       "      <td id=\"T_66779_row9_col5\" class=\"data row9 col5\" >0.8696</td>\n",
       "      <td id=\"T_66779_row9_col6\" class=\"data row9 col6\" >0.8455</td>\n",
       "      <td id=\"T_66779_row9_col7\" class=\"data row9 col7\" >0.8469</td>\n",
       "      <td id=\"T_66779_row9_col8\" class=\"data row9 col8\" >0.7822</td>\n",
       "      <td id=\"T_66779_row9_col9\" class=\"data row9 col9\" >0.1260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_66779_level0_row10\" class=\"row_heading level0 row10\" >knn</th>\n",
       "      <td id=\"T_66779_row10_col0\" class=\"data row10 col0\" >K Neighbors Classifier</td>\n",
       "      <td id=\"T_66779_row10_col1\" class=\"data row10 col1\" >0.8447</td>\n",
       "      <td id=\"T_66779_row10_col2\" class=\"data row10 col2\" >0.7198</td>\n",
       "      <td id=\"T_66779_row10_col3\" class=\"data row10 col3\" >0.2357</td>\n",
       "      <td id=\"T_66779_row10_col4\" class=\"data row10 col4\" >0.5102</td>\n",
       "      <td id=\"T_66779_row10_col5\" class=\"data row10 col5\" >0.3179</td>\n",
       "      <td id=\"T_66779_row10_col6\" class=\"data row10 col6\" >0.2437</td>\n",
       "      <td id=\"T_66779_row10_col7\" class=\"data row10 col7\" >0.2691</td>\n",
       "      <td id=\"T_66779_row10_col8\" class=\"data row10 col8\" >0.3693</td>\n",
       "      <td id=\"T_66779_row10_col9\" class=\"data row10 col9\" >0.2890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_66779_level0_row11\" class=\"row_heading level0 row11\" >dummy</th>\n",
       "      <td id=\"T_66779_row11_col0\" class=\"data row11 col0\" >Dummy Classifier</td>\n",
       "      <td id=\"T_66779_row11_col1\" class=\"data row11 col1\" >0.8454</td>\n",
       "      <td id=\"T_66779_row11_col2\" class=\"data row11 col2\" >0.5000</td>\n",
       "      <td id=\"T_66779_row11_col3\" class=\"data row11 col3\" >0.0000</td>\n",
       "      <td id=\"T_66779_row11_col4\" class=\"data row11 col4\" >0.0000</td>\n",
       "      <td id=\"T_66779_row11_col5\" class=\"data row11 col5\" >0.0000</td>\n",
       "      <td id=\"T_66779_row11_col6\" class=\"data row11 col6\" >0.0000</td>\n",
       "      <td id=\"T_66779_row11_col7\" class=\"data row11 col7\" >0.0000</td>\n",
       "      <td id=\"T_66779_row11_col8\" class=\"data row11 col8\" >0.1546</td>\n",
       "      <td id=\"T_66779_row11_col9\" class=\"data row11 col9\" >0.0370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_66779_level0_row12\" class=\"row_heading level0 row12\" >svm</th>\n",
       "      <td id=\"T_66779_row12_col0\" class=\"data row12 col0\" >SVM - Linear Kernel</td>\n",
       "      <td id=\"T_66779_row12_col1\" class=\"data row12 col1\" >0.7659</td>\n",
       "      <td id=\"T_66779_row12_col2\" class=\"data row12 col2\" >0.0000</td>\n",
       "      <td id=\"T_66779_row12_col3\" class=\"data row12 col3\" >0.1412</td>\n",
       "      <td id=\"T_66779_row12_col4\" class=\"data row12 col4\" >0.1539</td>\n",
       "      <td id=\"T_66779_row12_col5\" class=\"data row12 col5\" >0.0854</td>\n",
       "      <td id=\"T_66779_row12_col6\" class=\"data row12 col6\" >0.0296</td>\n",
       "      <td id=\"T_66779_row12_col7\" class=\"data row12 col7\" >0.0391</td>\n",
       "      <td id=\"T_66779_row12_col8\" class=\"data row12 col8\" >0.0000</td>\n",
       "      <td id=\"T_66779_row12_col9\" class=\"data row12 col9\" >0.0750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_66779_level0_row13\" class=\"row_heading level0 row13\" >ridge</th>\n",
       "      <td id=\"T_66779_row13_col0\" class=\"data row13 col0\" >Ridge Classifier</td>\n",
       "      <td id=\"T_66779_row13_col1\" class=\"data row13 col1\" >0.9521</td>\n",
       "      <td id=\"T_66779_row13_col2\" class=\"data row13 col2\" >0.0000</td>\n",
       "      <td id=\"T_66779_row13_col3\" class=\"data row13 col3\" >0.6908</td>\n",
       "      <td id=\"T_66779_row13_col4\" class=\"data row13 col4\" >1.0000</td>\n",
       "      <td id=\"T_66779_row13_col5\" class=\"data row13 col5\" >0.8134</td>\n",
       "      <td id=\"T_66779_row13_col6\" class=\"data row13 col6\" >0.7874</td>\n",
       "      <td id=\"T_66779_row13_col7\" class=\"data row13 col7\" >0.8072</td>\n",
       "      <td id=\"T_66779_row13_col8\" class=\"data row13 col8\" >0.0000</td>\n",
       "      <td id=\"T_66779_row13_col9\" class=\"data row13 col9\" >0.0530</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x1a612dc00a0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "clf1 = setup(data = df_train, \n",
    "             target = 'target',\n",
    "             numeric_features=numeric_feature_list,\n",
    "             silent=True,\n",
    "             log_experiment = True,\n",
    "             use_gpu=False,\n",
    "             experiment_name = 'feature selection',\n",
    "             feature_selection = True, feature_selection_threshold = 0.5,\n",
    "            )\n",
    "add_metric('apc', 'APC', average_precision_score, target = 'pred_proba')\n",
    "best = compare_models(sort=\"APC\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Removing multicollinearity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_36c8d th {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_36c8d_row0_col0, #T_36c8d_row0_col3, #T_36c8d_row0_col4, #T_36c8d_row1_col0, #T_36c8d_row1_col1, #T_36c8d_row1_col2, #T_36c8d_row1_col3, #T_36c8d_row1_col4, #T_36c8d_row1_col5, #T_36c8d_row1_col6, #T_36c8d_row1_col7, #T_36c8d_row1_col8, #T_36c8d_row2_col0, #T_36c8d_row2_col1, #T_36c8d_row2_col2, #T_36c8d_row2_col3, #T_36c8d_row2_col4, #T_36c8d_row2_col5, #T_36c8d_row2_col6, #T_36c8d_row2_col7, #T_36c8d_row2_col8, #T_36c8d_row3_col0, #T_36c8d_row3_col1, #T_36c8d_row3_col2, #T_36c8d_row3_col3, #T_36c8d_row3_col4, #T_36c8d_row3_col5, #T_36c8d_row3_col6, #T_36c8d_row3_col7, #T_36c8d_row3_col8, #T_36c8d_row4_col0, #T_36c8d_row4_col1, #T_36c8d_row4_col2, #T_36c8d_row4_col3, #T_36c8d_row4_col4, #T_36c8d_row4_col5, #T_36c8d_row4_col6, #T_36c8d_row4_col7, #T_36c8d_row4_col8, #T_36c8d_row5_col0, #T_36c8d_row5_col1, #T_36c8d_row5_col2, #T_36c8d_row5_col3, #T_36c8d_row5_col4, #T_36c8d_row5_col5, #T_36c8d_row5_col6, #T_36c8d_row5_col7, #T_36c8d_row5_col8, #T_36c8d_row6_col0, #T_36c8d_row6_col1, #T_36c8d_row6_col2, #T_36c8d_row6_col3, #T_36c8d_row6_col4, #T_36c8d_row6_col5, #T_36c8d_row6_col6, #T_36c8d_row6_col7, #T_36c8d_row6_col8, #T_36c8d_row7_col0, #T_36c8d_row7_col1, #T_36c8d_row7_col2, #T_36c8d_row7_col3, #T_36c8d_row7_col4, #T_36c8d_row7_col5, #T_36c8d_row7_col6, #T_36c8d_row7_col7, #T_36c8d_row7_col8, #T_36c8d_row8_col0, #T_36c8d_row8_col1, #T_36c8d_row8_col2, #T_36c8d_row8_col4, #T_36c8d_row8_col5, #T_36c8d_row8_col6, #T_36c8d_row8_col7, #T_36c8d_row8_col8, #T_36c8d_row9_col0, #T_36c8d_row9_col1, #T_36c8d_row9_col2, #T_36c8d_row9_col3, #T_36c8d_row9_col4, #T_36c8d_row9_col5, #T_36c8d_row9_col6, #T_36c8d_row9_col7, #T_36c8d_row9_col8, #T_36c8d_row10_col0, #T_36c8d_row10_col1, #T_36c8d_row10_col2, #T_36c8d_row10_col3, #T_36c8d_row10_col4, #T_36c8d_row10_col5, #T_36c8d_row10_col6, #T_36c8d_row10_col7, #T_36c8d_row10_col8, #T_36c8d_row11_col0, #T_36c8d_row11_col1, #T_36c8d_row11_col2, #T_36c8d_row11_col3, #T_36c8d_row11_col4, #T_36c8d_row11_col5, #T_36c8d_row11_col6, #T_36c8d_row11_col7, #T_36c8d_row11_col8, #T_36c8d_row12_col0, #T_36c8d_row12_col1, #T_36c8d_row12_col2, #T_36c8d_row12_col3, #T_36c8d_row12_col4, #T_36c8d_row12_col5, #T_36c8d_row12_col6, #T_36c8d_row12_col7, #T_36c8d_row12_col8, #T_36c8d_row13_col0, #T_36c8d_row13_col1, #T_36c8d_row13_col2, #T_36c8d_row13_col3, #T_36c8d_row13_col5, #T_36c8d_row13_col6, #T_36c8d_row13_col7, #T_36c8d_row13_col8 {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_36c8d_row0_col1, #T_36c8d_row0_col2, #T_36c8d_row0_col5, #T_36c8d_row0_col6, #T_36c8d_row0_col7, #T_36c8d_row0_col8, #T_36c8d_row8_col3, #T_36c8d_row13_col4 {\n",
       "  text-align: left;\n",
       "  background-color: yellow;\n",
       "}\n",
       "#T_36c8d_row0_col9, #T_36c8d_row1_col9, #T_36c8d_row2_col9, #T_36c8d_row3_col9, #T_36c8d_row4_col9, #T_36c8d_row5_col9, #T_36c8d_row6_col9, #T_36c8d_row7_col9, #T_36c8d_row8_col9, #T_36c8d_row9_col9, #T_36c8d_row10_col9, #T_36c8d_row11_col9, #T_36c8d_row12_col9 {\n",
       "  text-align: left;\n",
       "  background-color: lightgrey;\n",
       "}\n",
       "#T_36c8d_row13_col9 {\n",
       "  text-align: left;\n",
       "  background-color: yellow;\n",
       "  background-color: lightgrey;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_36c8d\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_36c8d_level0_col0\" class=\"col_heading level0 col0\" >Model</th>\n",
       "      <th id=\"T_36c8d_level0_col1\" class=\"col_heading level0 col1\" >Accuracy</th>\n",
       "      <th id=\"T_36c8d_level0_col2\" class=\"col_heading level0 col2\" >AUC</th>\n",
       "      <th id=\"T_36c8d_level0_col3\" class=\"col_heading level0 col3\" >Recall</th>\n",
       "      <th id=\"T_36c8d_level0_col4\" class=\"col_heading level0 col4\" >Prec.</th>\n",
       "      <th id=\"T_36c8d_level0_col5\" class=\"col_heading level0 col5\" >F1</th>\n",
       "      <th id=\"T_36c8d_level0_col6\" class=\"col_heading level0 col6\" >Kappa</th>\n",
       "      <th id=\"T_36c8d_level0_col7\" class=\"col_heading level0 col7\" >MCC</th>\n",
       "      <th id=\"T_36c8d_level0_col8\" class=\"col_heading level0 col8\" >APC</th>\n",
       "      <th id=\"T_36c8d_level0_col9\" class=\"col_heading level0 col9\" >TT (Sec)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_36c8d_level0_row0\" class=\"row_heading level0 row0\" >lightgbm</th>\n",
       "      <td id=\"T_36c8d_row0_col0\" class=\"data row0 col0\" >Light Gradient Boosting Machine</td>\n",
       "      <td id=\"T_36c8d_row0_col1\" class=\"data row0 col1\" >0.9770</td>\n",
       "      <td id=\"T_36c8d_row0_col2\" class=\"data row0 col2\" >0.9739</td>\n",
       "      <td id=\"T_36c8d_row0_col3\" class=\"data row0 col3\" >0.8545</td>\n",
       "      <td id=\"T_36c8d_row0_col4\" class=\"data row0 col4\" >0.9747</td>\n",
       "      <td id=\"T_36c8d_row0_col5\" class=\"data row0 col5\" >0.9084</td>\n",
       "      <td id=\"T_36c8d_row0_col6\" class=\"data row0 col6\" >0.8954</td>\n",
       "      <td id=\"T_36c8d_row0_col7\" class=\"data row0 col7\" >0.8993</td>\n",
       "      <td id=\"T_36c8d_row0_col8\" class=\"data row0 col8\" >0.9450</td>\n",
       "      <td id=\"T_36c8d_row0_col9\" class=\"data row0 col9\" >0.7050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_36c8d_level0_row1\" class=\"row_heading level0 row1\" >et</th>\n",
       "      <td id=\"T_36c8d_row1_col0\" class=\"data row1 col0\" >Extra Trees Classifier</td>\n",
       "      <td id=\"T_36c8d_row1_col1\" class=\"data row1 col1\" >0.9757</td>\n",
       "      <td id=\"T_36c8d_row1_col2\" class=\"data row1 col2\" >0.9734</td>\n",
       "      <td id=\"T_36c8d_row1_col3\" class=\"data row1 col3\" >0.8407</td>\n",
       "      <td id=\"T_36c8d_row1_col4\" class=\"data row1 col4\" >0.9792</td>\n",
       "      <td id=\"T_36c8d_row1_col5\" class=\"data row1 col5\" >0.9017</td>\n",
       "      <td id=\"T_36c8d_row1_col6\" class=\"data row1 col6\" >0.8881</td>\n",
       "      <td id=\"T_36c8d_row1_col7\" class=\"data row1 col7\" >0.8932</td>\n",
       "      <td id=\"T_36c8d_row1_col8\" class=\"data row1 col8\" >0.9436</td>\n",
       "      <td id=\"T_36c8d_row1_col9\" class=\"data row1 col9\" >1.0790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_36c8d_level0_row2\" class=\"row_heading level0 row2\" >rf</th>\n",
       "      <td id=\"T_36c8d_row2_col0\" class=\"data row2 col0\" >Random Forest Classifier</td>\n",
       "      <td id=\"T_36c8d_row2_col1\" class=\"data row2 col1\" >0.9738</td>\n",
       "      <td id=\"T_36c8d_row2_col2\" class=\"data row2 col2\" >0.9697</td>\n",
       "      <td id=\"T_36c8d_row2_col3\" class=\"data row2 col3\" >0.8457</td>\n",
       "      <td id=\"T_36c8d_row2_col4\" class=\"data row2 col4\" >0.9596</td>\n",
       "      <td id=\"T_36c8d_row2_col5\" class=\"data row2 col5\" >0.8961</td>\n",
       "      <td id=\"T_36c8d_row2_col6\" class=\"data row2 col6\" >0.8813</td>\n",
       "      <td id=\"T_36c8d_row2_col7\" class=\"data row2 col7\" >0.8854</td>\n",
       "      <td id=\"T_36c8d_row2_col8\" class=\"data row2 col8\" >0.9379</td>\n",
       "      <td id=\"T_36c8d_row2_col9\" class=\"data row2 col9\" >1.9970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_36c8d_level0_row3\" class=\"row_heading level0 row3\" >gbc</th>\n",
       "      <td id=\"T_36c8d_row3_col0\" class=\"data row3 col0\" >Gradient Boosting Classifier</td>\n",
       "      <td id=\"T_36c8d_row3_col1\" class=\"data row3 col1\" >0.9700</td>\n",
       "      <td id=\"T_36c8d_row3_col2\" class=\"data row3 col2\" >0.9645</td>\n",
       "      <td id=\"T_36c8d_row3_col3\" class=\"data row3 col3\" >0.8639</td>\n",
       "      <td id=\"T_36c8d_row3_col4\" class=\"data row3 col4\" >0.9157</td>\n",
       "      <td id=\"T_36c8d_row3_col5\" class=\"data row3 col5\" >0.8855</td>\n",
       "      <td id=\"T_36c8d_row3_col6\" class=\"data row3 col6\" >0.8683</td>\n",
       "      <td id=\"T_36c8d_row3_col7\" class=\"data row3 col7\" >0.8710</td>\n",
       "      <td id=\"T_36c8d_row3_col8\" class=\"data row3 col8\" >0.9298</td>\n",
       "      <td id=\"T_36c8d_row3_col9\" class=\"data row3 col9\" >2.7430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_36c8d_level0_row4\" class=\"row_heading level0 row4\" >lda</th>\n",
       "      <td id=\"T_36c8d_row4_col0\" class=\"data row4 col0\" >Linear Discriminant Analysis</td>\n",
       "      <td id=\"T_36c8d_row4_col1\" class=\"data row4 col1\" >0.9674</td>\n",
       "      <td id=\"T_36c8d_row4_col2\" class=\"data row4 col2\" >0.9591</td>\n",
       "      <td id=\"T_36c8d_row4_col3\" class=\"data row4 col3\" >0.7749</td>\n",
       "      <td id=\"T_36c8d_row4_col4\" class=\"data row4 col4\" >0.9827</td>\n",
       "      <td id=\"T_36c8d_row4_col5\" class=\"data row4 col5\" >0.8625</td>\n",
       "      <td id=\"T_36c8d_row4_col6\" class=\"data row4 col6\" >0.8446</td>\n",
       "      <td id=\"T_36c8d_row4_col7\" class=\"data row4 col7\" >0.8545</td>\n",
       "      <td id=\"T_36c8d_row4_col8\" class=\"data row4 col8\" >0.9277</td>\n",
       "      <td id=\"T_36c8d_row4_col9\" class=\"data row4 col9\" >0.0720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_36c8d_level0_row5\" class=\"row_heading level0 row5\" >ada</th>\n",
       "      <td id=\"T_36c8d_row5_col0\" class=\"data row5 col0\" >Ada Boost Classifier</td>\n",
       "      <td id=\"T_36c8d_row5_col1\" class=\"data row5 col1\" >0.9732</td>\n",
       "      <td id=\"T_36c8d_row5_col2\" class=\"data row5 col2\" >0.9489</td>\n",
       "      <td id=\"T_36c8d_row5_col3\" class=\"data row5 col3\" >0.8541</td>\n",
       "      <td id=\"T_36c8d_row5_col4\" class=\"data row5 col4\" >0.9439</td>\n",
       "      <td id=\"T_36c8d_row5_col5\" class=\"data row5 col5\" >0.8946</td>\n",
       "      <td id=\"T_36c8d_row5_col6\" class=\"data row5 col6\" >0.8794</td>\n",
       "      <td id=\"T_36c8d_row5_col7\" class=\"data row5 col7\" >0.8821</td>\n",
       "      <td id=\"T_36c8d_row5_col8\" class=\"data row5 col8\" >0.9147</td>\n",
       "      <td id=\"T_36c8d_row5_col9\" class=\"data row5 col9\" >0.7530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_36c8d_level0_row6\" class=\"row_heading level0 row6\" >lr</th>\n",
       "      <td id=\"T_36c8d_row6_col0\" class=\"data row6 col0\" >Logistic Regression</td>\n",
       "      <td id=\"T_36c8d_row6_col1\" class=\"data row6 col1\" >0.9629</td>\n",
       "      <td id=\"T_36c8d_row6_col2\" class=\"data row6 col2\" >0.9460</td>\n",
       "      <td id=\"T_36c8d_row6_col3\" class=\"data row6 col3\" >0.8221</td>\n",
       "      <td id=\"T_36c8d_row6_col4\" class=\"data row6 col4\" >0.9022</td>\n",
       "      <td id=\"T_36c8d_row6_col5\" class=\"data row6 col5\" >0.8571</td>\n",
       "      <td id=\"T_36c8d_row6_col6\" class=\"data row6 col6\" >0.8360</td>\n",
       "      <td id=\"T_36c8d_row6_col7\" class=\"data row6 col7\" >0.8391</td>\n",
       "      <td id=\"T_36c8d_row6_col8\" class=\"data row6 col8\" >0.9055</td>\n",
       "      <td id=\"T_36c8d_row6_col9\" class=\"data row6 col9\" >0.1200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_36c8d_level0_row7\" class=\"row_heading level0 row7\" >nb</th>\n",
       "      <td id=\"T_36c8d_row7_col0\" class=\"data row7 col0\" >Naive Bayes</td>\n",
       "      <td id=\"T_36c8d_row7_col1\" class=\"data row7 col1\" >0.9431</td>\n",
       "      <td id=\"T_36c8d_row7_col2\" class=\"data row7 col2\" >0.9611</td>\n",
       "      <td id=\"T_36c8d_row7_col3\" class=\"data row7 col3\" >0.6574</td>\n",
       "      <td id=\"T_36c8d_row7_col4\" class=\"data row7 col4\" >0.9035</td>\n",
       "      <td id=\"T_36c8d_row7_col5\" class=\"data row7 col5\" >0.7543</td>\n",
       "      <td id=\"T_36c8d_row7_col6\" class=\"data row7 col6\" >0.7236</td>\n",
       "      <td id=\"T_36c8d_row7_col7\" class=\"data row7 col7\" >0.7389</td>\n",
       "      <td id=\"T_36c8d_row7_col8\" class=\"data row7 col8\" >0.8741</td>\n",
       "      <td id=\"T_36c8d_row7_col9\" class=\"data row7 col9\" >0.0260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_36c8d_level0_row8\" class=\"row_heading level0 row8\" >qda</th>\n",
       "      <td id=\"T_36c8d_row8_col0\" class=\"data row8 col0\" >Quadratic Discriminant Analysis</td>\n",
       "      <td id=\"T_36c8d_row8_col1\" class=\"data row8 col1\" >0.9476</td>\n",
       "      <td id=\"T_36c8d_row8_col2\" class=\"data row8 col2\" >0.9611</td>\n",
       "      <td id=\"T_36c8d_row8_col3\" class=\"data row8 col3\" >0.8734</td>\n",
       "      <td id=\"T_36c8d_row8_col4\" class=\"data row8 col4\" >0.7751</td>\n",
       "      <td id=\"T_36c8d_row8_col5\" class=\"data row8 col5\" >0.8200</td>\n",
       "      <td id=\"T_36c8d_row8_col6\" class=\"data row8 col6\" >0.7895</td>\n",
       "      <td id=\"T_36c8d_row8_col7\" class=\"data row8 col7\" >0.7923</td>\n",
       "      <td id=\"T_36c8d_row8_col8\" class=\"data row8 col8\" >0.8121</td>\n",
       "      <td id=\"T_36c8d_row8_col9\" class=\"data row8 col9\" >0.0640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_36c8d_level0_row9\" class=\"row_heading level0 row9\" >dt</th>\n",
       "      <td id=\"T_36c8d_row9_col0\" class=\"data row9 col0\" >Decision Tree Classifier</td>\n",
       "      <td id=\"T_36c8d_row9_col1\" class=\"data row9 col1\" >0.9540</td>\n",
       "      <td id=\"T_36c8d_row9_col2\" class=\"data row9 col2\" >0.9121</td>\n",
       "      <td id=\"T_36c8d_row9_col3\" class=\"data row9 col3\" >0.8545</td>\n",
       "      <td id=\"T_36c8d_row9_col4\" class=\"data row9 col4\" >0.8238</td>\n",
       "      <td id=\"T_36c8d_row9_col5\" class=\"data row9 col5\" >0.8355</td>\n",
       "      <td id=\"T_36c8d_row9_col6\" class=\"data row9 col6\" >0.8089</td>\n",
       "      <td id=\"T_36c8d_row9_col7\" class=\"data row9 col7\" >0.8113</td>\n",
       "      <td id=\"T_36c8d_row9_col8\" class=\"data row9 col8\" >0.7223</td>\n",
       "      <td id=\"T_36c8d_row9_col9\" class=\"data row9 col9\" >0.0650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_36c8d_level0_row10\" class=\"row_heading level0 row10\" >knn</th>\n",
       "      <td id=\"T_36c8d_row10_col0\" class=\"data row10 col0\" >K Neighbors Classifier</td>\n",
       "      <td id=\"T_36c8d_row10_col1\" class=\"data row10 col1\" >0.8569</td>\n",
       "      <td id=\"T_36c8d_row10_col2\" class=\"data row10 col2\" >0.7044</td>\n",
       "      <td id=\"T_36c8d_row10_col3\" class=\"data row10 col3\" >0.2299</td>\n",
       "      <td id=\"T_36c8d_row10_col4\" class=\"data row10 col4\" >0.4565</td>\n",
       "      <td id=\"T_36c8d_row10_col5\" class=\"data row10 col5\" >0.3001</td>\n",
       "      <td id=\"T_36c8d_row10_col6\" class=\"data row10 col6\" >0.2312</td>\n",
       "      <td id=\"T_36c8d_row10_col7\" class=\"data row10 col7\" >0.2504</td>\n",
       "      <td id=\"T_36c8d_row10_col8\" class=\"data row10 col8\" >0.3057</td>\n",
       "      <td id=\"T_36c8d_row10_col9\" class=\"data row10 col9\" >0.0660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_36c8d_level0_row11\" class=\"row_heading level0 row11\" >dummy</th>\n",
       "      <td id=\"T_36c8d_row11_col0\" class=\"data row11 col0\" >Dummy Classifier</td>\n",
       "      <td id=\"T_36c8d_row11_col1\" class=\"data row11 col1\" >0.8639</td>\n",
       "      <td id=\"T_36c8d_row11_col2\" class=\"data row11 col2\" >0.5000</td>\n",
       "      <td id=\"T_36c8d_row11_col3\" class=\"data row11 col3\" >0.0000</td>\n",
       "      <td id=\"T_36c8d_row11_col4\" class=\"data row11 col4\" >0.0000</td>\n",
       "      <td id=\"T_36c8d_row11_col5\" class=\"data row11 col5\" >0.0000</td>\n",
       "      <td id=\"T_36c8d_row11_col6\" class=\"data row11 col6\" >0.0000</td>\n",
       "      <td id=\"T_36c8d_row11_col7\" class=\"data row11 col7\" >0.0000</td>\n",
       "      <td id=\"T_36c8d_row11_col8\" class=\"data row11 col8\" >0.1361</td>\n",
       "      <td id=\"T_36c8d_row11_col9\" class=\"data row11 col9\" >0.0340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_36c8d_level0_row12\" class=\"row_heading level0 row12\" >svm</th>\n",
       "      <td id=\"T_36c8d_row12_col0\" class=\"data row12 col0\" >SVM - Linear Kernel</td>\n",
       "      <td id=\"T_36c8d_row12_col1\" class=\"data row12 col1\" >0.7158</td>\n",
       "      <td id=\"T_36c8d_row12_col2\" class=\"data row12 col2\" >0.0000</td>\n",
       "      <td id=\"T_36c8d_row12_col3\" class=\"data row12 col3\" >0.2000</td>\n",
       "      <td id=\"T_36c8d_row12_col4\" class=\"data row12 col4\" >0.0269</td>\n",
       "      <td id=\"T_36c8d_row12_col5\" class=\"data row12 col5\" >0.0475</td>\n",
       "      <td id=\"T_36c8d_row12_col6\" class=\"data row12 col6\" >-0.0037</td>\n",
       "      <td id=\"T_36c8d_row12_col7\" class=\"data row12 col7\" >-0.0095</td>\n",
       "      <td id=\"T_36c8d_row12_col8\" class=\"data row12 col8\" >0.0000</td>\n",
       "      <td id=\"T_36c8d_row12_col9\" class=\"data row12 col9\" >0.0420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_36c8d_level0_row13\" class=\"row_heading level0 row13\" >ridge</th>\n",
       "      <td id=\"T_36c8d_row13_col0\" class=\"data row13 col0\" >Ridge Classifier</td>\n",
       "      <td id=\"T_36c8d_row13_col1\" class=\"data row13 col1\" >0.9610</td>\n",
       "      <td id=\"T_36c8d_row13_col2\" class=\"data row13 col2\" >0.0000</td>\n",
       "      <td id=\"T_36c8d_row13_col3\" class=\"data row13 col3\" >0.7236</td>\n",
       "      <td id=\"T_36c8d_row13_col4\" class=\"data row13 col4\" >0.9871</td>\n",
       "      <td id=\"T_36c8d_row13_col5\" class=\"data row13 col5\" >0.8278</td>\n",
       "      <td id=\"T_36c8d_row13_col6\" class=\"data row13 col6\" >0.8071</td>\n",
       "      <td id=\"T_36c8d_row13_col7\" class=\"data row13 col7\" >0.8232</td>\n",
       "      <td id=\"T_36c8d_row13_col8\" class=\"data row13 col8\" >0.0000</td>\n",
       "      <td id=\"T_36c8d_row13_col9\" class=\"data row13 col9\" >0.0210</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x1a61022ce80>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "clf1 = setup(data = df_train, \n",
    "             target = 'target', \n",
    "             numeric_features=numeric_feature_list,\n",
    "             silent=True,\n",
    "             log_experiment = True,\n",
    "             use_gpu=False,\n",
    "             experiment_name = 'removing multicollinearity',\n",
    "             remove_multicollinearity = True, multicollinearity_threshold = 0.6,\n",
    "            )\n",
    "add_metric('apc', 'APC', average_precision_score, target = 'pred_proba')\n",
    "best = compare_models(sort=\"APC\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_740b5 th {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_740b5_row0_col0, #T_740b5_row0_col1, #T_740b5_row0_col2, #T_740b5_row0_col3, #T_740b5_row0_col4, #T_740b5_row0_col5, #T_740b5_row0_col6, #T_740b5_row0_col7, #T_740b5_row1_col0, #T_740b5_row1_col1, #T_740b5_row1_col3, #T_740b5_row1_col4, #T_740b5_row1_col5, #T_740b5_row1_col6, #T_740b5_row1_col7, #T_740b5_row1_col8, #T_740b5_row2_col0, #T_740b5_row2_col1, #T_740b5_row2_col2, #T_740b5_row2_col3, #T_740b5_row2_col4, #T_740b5_row2_col5, #T_740b5_row2_col6, #T_740b5_row2_col7, #T_740b5_row2_col8, #T_740b5_row3_col0, #T_740b5_row3_col2, #T_740b5_row3_col3, #T_740b5_row3_col4, #T_740b5_row3_col8, #T_740b5_row4_col0, #T_740b5_row4_col1, #T_740b5_row4_col2, #T_740b5_row4_col3, #T_740b5_row4_col4, #T_740b5_row4_col5, #T_740b5_row4_col6, #T_740b5_row4_col7, #T_740b5_row4_col8, #T_740b5_row5_col0, #T_740b5_row5_col1, #T_740b5_row5_col2, #T_740b5_row5_col3, #T_740b5_row5_col4, #T_740b5_row5_col5, #T_740b5_row5_col6, #T_740b5_row5_col7, #T_740b5_row5_col8, #T_740b5_row6_col0, #T_740b5_row6_col1, #T_740b5_row6_col2, #T_740b5_row6_col3, #T_740b5_row6_col4, #T_740b5_row6_col5, #T_740b5_row6_col6, #T_740b5_row6_col7, #T_740b5_row6_col8, #T_740b5_row7_col0, #T_740b5_row7_col1, #T_740b5_row7_col2, #T_740b5_row7_col3, #T_740b5_row7_col4, #T_740b5_row7_col5, #T_740b5_row7_col6, #T_740b5_row7_col7, #T_740b5_row7_col8, #T_740b5_row8_col0, #T_740b5_row8_col1, #T_740b5_row8_col2, #T_740b5_row8_col3, #T_740b5_row8_col4, #T_740b5_row8_col5, #T_740b5_row8_col6, #T_740b5_row8_col7, #T_740b5_row8_col8, #T_740b5_row9_col0, #T_740b5_row9_col1, #T_740b5_row9_col2, #T_740b5_row9_col4, #T_740b5_row9_col5, #T_740b5_row9_col6, #T_740b5_row9_col7, #T_740b5_row9_col8, #T_740b5_row10_col0, #T_740b5_row10_col1, #T_740b5_row10_col2, #T_740b5_row10_col3, #T_740b5_row10_col4, #T_740b5_row10_col5, #T_740b5_row10_col6, #T_740b5_row10_col7, #T_740b5_row10_col8, #T_740b5_row11_col0, #T_740b5_row11_col1, #T_740b5_row11_col2, #T_740b5_row11_col3, #T_740b5_row11_col4, #T_740b5_row11_col5, #T_740b5_row11_col6, #T_740b5_row11_col7, #T_740b5_row11_col8, #T_740b5_row12_col0, #T_740b5_row12_col1, #T_740b5_row12_col2, #T_740b5_row12_col3, #T_740b5_row12_col4, #T_740b5_row12_col5, #T_740b5_row12_col6, #T_740b5_row12_col7, #T_740b5_row12_col8, #T_740b5_row13_col0, #T_740b5_row13_col1, #T_740b5_row13_col2, #T_740b5_row13_col3, #T_740b5_row13_col5, #T_740b5_row13_col6, #T_740b5_row13_col7, #T_740b5_row13_col8 {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_740b5_row0_col8, #T_740b5_row1_col2, #T_740b5_row3_col1, #T_740b5_row3_col5, #T_740b5_row3_col6, #T_740b5_row3_col7, #T_740b5_row9_col3, #T_740b5_row13_col4 {\n",
       "  text-align: left;\n",
       "  background-color: yellow;\n",
       "}\n",
       "#T_740b5_row0_col9, #T_740b5_row1_col9, #T_740b5_row2_col9, #T_740b5_row3_col9, #T_740b5_row4_col9, #T_740b5_row5_col9, #T_740b5_row6_col9, #T_740b5_row7_col9, #T_740b5_row8_col9, #T_740b5_row9_col9, #T_740b5_row10_col9, #T_740b5_row12_col9, #T_740b5_row13_col9 {\n",
       "  text-align: left;\n",
       "  background-color: lightgrey;\n",
       "}\n",
       "#T_740b5_row11_col9 {\n",
       "  text-align: left;\n",
       "  background-color: yellow;\n",
       "  background-color: lightgrey;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_740b5\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_740b5_level0_col0\" class=\"col_heading level0 col0\" >Model</th>\n",
       "      <th id=\"T_740b5_level0_col1\" class=\"col_heading level0 col1\" >Accuracy</th>\n",
       "      <th id=\"T_740b5_level0_col2\" class=\"col_heading level0 col2\" >AUC</th>\n",
       "      <th id=\"T_740b5_level0_col3\" class=\"col_heading level0 col3\" >Recall</th>\n",
       "      <th id=\"T_740b5_level0_col4\" class=\"col_heading level0 col4\" >Prec.</th>\n",
       "      <th id=\"T_740b5_level0_col5\" class=\"col_heading level0 col5\" >F1</th>\n",
       "      <th id=\"T_740b5_level0_col6\" class=\"col_heading level0 col6\" >Kappa</th>\n",
       "      <th id=\"T_740b5_level0_col7\" class=\"col_heading level0 col7\" >MCC</th>\n",
       "      <th id=\"T_740b5_level0_col8\" class=\"col_heading level0 col8\" >APC</th>\n",
       "      <th id=\"T_740b5_level0_col9\" class=\"col_heading level0 col9\" >TT (Sec)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_740b5_level0_row0\" class=\"row_heading level0 row0\" >lr</th>\n",
       "      <td id=\"T_740b5_row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
       "      <td id=\"T_740b5_row0_col1\" class=\"data row0 col1\" >0.9776</td>\n",
       "      <td id=\"T_740b5_row0_col2\" class=\"data row0 col2\" >0.9806</td>\n",
       "      <td id=\"T_740b5_row0_col3\" class=\"data row0 col3\" >0.8708</td>\n",
       "      <td id=\"T_740b5_row0_col4\" class=\"data row0 col4\" >0.9668</td>\n",
       "      <td id=\"T_740b5_row0_col5\" class=\"data row0 col5\" >0.9129</td>\n",
       "      <td id=\"T_740b5_row0_col6\" class=\"data row0 col6\" >0.9002</td>\n",
       "      <td id=\"T_740b5_row0_col7\" class=\"data row0 col7\" >0.9039</td>\n",
       "      <td id=\"T_740b5_row0_col8\" class=\"data row0 col8\" >0.9526</td>\n",
       "      <td id=\"T_740b5_row0_col9\" class=\"data row0 col9\" >0.2910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_740b5_level0_row1\" class=\"row_heading level0 row1\" >lda</th>\n",
       "      <td id=\"T_740b5_row1_col0\" class=\"data row1 col0\" >Linear Discriminant Analysis</td>\n",
       "      <td id=\"T_740b5_row1_col1\" class=\"data row1 col1\" >0.9591</td>\n",
       "      <td id=\"T_740b5_row1_col2\" class=\"data row1 col2\" >0.9818</td>\n",
       "      <td id=\"T_740b5_row1_col3\" class=\"data row1 col3\" >0.7089</td>\n",
       "      <td id=\"T_740b5_row1_col4\" class=\"data row1 col4\" >0.9947</td>\n",
       "      <td id=\"T_740b5_row1_col5\" class=\"data row1 col5\" >0.8250</td>\n",
       "      <td id=\"T_740b5_row1_col6\" class=\"data row1 col6\" >0.8028</td>\n",
       "      <td id=\"T_740b5_row1_col7\" class=\"data row1 col7\" >0.8191</td>\n",
       "      <td id=\"T_740b5_row1_col8\" class=\"data row1 col8\" >0.9492</td>\n",
       "      <td id=\"T_740b5_row1_col9\" class=\"data row1 col9\" >0.0390</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_740b5_level0_row2\" class=\"row_heading level0 row2\" >gbc</th>\n",
       "      <td id=\"T_740b5_row2_col0\" class=\"data row2 col0\" >Gradient Boosting Classifier</td>\n",
       "      <td id=\"T_740b5_row2_col1\" class=\"data row2 col1\" >0.9757</td>\n",
       "      <td id=\"T_740b5_row2_col2\" class=\"data row2 col2\" >0.9768</td>\n",
       "      <td id=\"T_740b5_row2_col3\" class=\"data row2 col3\" >0.8617</td>\n",
       "      <td id=\"T_740b5_row2_col4\" class=\"data row2 col4\" >0.9595</td>\n",
       "      <td id=\"T_740b5_row2_col5\" class=\"data row2 col5\" >0.9055</td>\n",
       "      <td id=\"T_740b5_row2_col6\" class=\"data row2 col6\" >0.8917</td>\n",
       "      <td id=\"T_740b5_row2_col7\" class=\"data row2 col7\" >0.8949</td>\n",
       "      <td id=\"T_740b5_row2_col8\" class=\"data row2 col8\" >0.9482</td>\n",
       "      <td id=\"T_740b5_row2_col9\" class=\"data row2 col9\" >1.8130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_740b5_level0_row3\" class=\"row_heading level0 row3\" >et</th>\n",
       "      <td id=\"T_740b5_row3_col0\" class=\"data row3 col0\" >Extra Trees Classifier</td>\n",
       "      <td id=\"T_740b5_row3_col1\" class=\"data row3 col1\" >0.9789</td>\n",
       "      <td id=\"T_740b5_row3_col2\" class=\"data row3 col2\" >0.9776</td>\n",
       "      <td id=\"T_740b5_row3_col3\" class=\"data row3 col3\" >0.8662</td>\n",
       "      <td id=\"T_740b5_row3_col4\" class=\"data row3 col4\" >0.9807</td>\n",
       "      <td id=\"T_740b5_row3_col5\" class=\"data row3 col5\" >0.9167</td>\n",
       "      <td id=\"T_740b5_row3_col6\" class=\"data row3 col6\" >0.9048</td>\n",
       "      <td id=\"T_740b5_row3_col7\" class=\"data row3 col7\" >0.9091</td>\n",
       "      <td id=\"T_740b5_row3_col8\" class=\"data row3 col8\" >0.9482</td>\n",
       "      <td id=\"T_740b5_row3_col9\" class=\"data row3 col9\" >0.8870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_740b5_level0_row4\" class=\"row_heading level0 row4\" >rf</th>\n",
       "      <td id=\"T_740b5_row4_col0\" class=\"data row4 col0\" >Random Forest Classifier</td>\n",
       "      <td id=\"T_740b5_row4_col1\" class=\"data row4 col1\" >0.9757</td>\n",
       "      <td id=\"T_740b5_row4_col2\" class=\"data row4 col2\" >0.9718</td>\n",
       "      <td id=\"T_740b5_row4_col3\" class=\"data row4 col3\" >0.8617</td>\n",
       "      <td id=\"T_740b5_row4_col4\" class=\"data row4 col4\" >0.9602</td>\n",
       "      <td id=\"T_740b5_row4_col5\" class=\"data row4 col5\" >0.9056</td>\n",
       "      <td id=\"T_740b5_row4_col6\" class=\"data row4 col6\" >0.8918</td>\n",
       "      <td id=\"T_740b5_row4_col7\" class=\"data row4 col7\" >0.8952</td>\n",
       "      <td id=\"T_740b5_row4_col8\" class=\"data row4 col8\" >0.9463</td>\n",
       "      <td id=\"T_740b5_row4_col9\" class=\"data row4 col9\" >1.4520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_740b5_level0_row5\" class=\"row_heading level0 row5\" >lightgbm</th>\n",
       "      <td id=\"T_740b5_row5_col0\" class=\"data row5 col0\" >Light Gradient Boosting Machine</td>\n",
       "      <td id=\"T_740b5_row5_col1\" class=\"data row5 col1\" >0.9764</td>\n",
       "      <td id=\"T_740b5_row5_col2\" class=\"data row5 col2\" >0.9708</td>\n",
       "      <td id=\"T_740b5_row5_col3\" class=\"data row5 col3\" >0.8617</td>\n",
       "      <td id=\"T_740b5_row5_col4\" class=\"data row5 col4\" >0.9655</td>\n",
       "      <td id=\"T_740b5_row5_col5\" class=\"data row5 col5\" >0.9077</td>\n",
       "      <td id=\"T_740b5_row5_col6\" class=\"data row5 col6\" >0.8943</td>\n",
       "      <td id=\"T_740b5_row5_col7\" class=\"data row5 col7\" >0.8980</td>\n",
       "      <td id=\"T_740b5_row5_col8\" class=\"data row5 col8\" >0.9380</td>\n",
       "      <td id=\"T_740b5_row5_col9\" class=\"data row5 col9\" >0.2740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_740b5_level0_row6\" class=\"row_heading level0 row6\" >ada</th>\n",
       "      <td id=\"T_740b5_row6_col0\" class=\"data row6 col0\" >Ada Boost Classifier</td>\n",
       "      <td id=\"T_740b5_row6_col1\" class=\"data row6 col1\" >0.9713</td>\n",
       "      <td id=\"T_740b5_row6_col2\" class=\"data row6 col2\" >0.9622</td>\n",
       "      <td id=\"T_740b5_row6_col3\" class=\"data row6 col3\" >0.8617</td>\n",
       "      <td id=\"T_740b5_row6_col4\" class=\"data row6 col4\" >0.9290</td>\n",
       "      <td id=\"T_740b5_row6_col5\" class=\"data row6 col5\" >0.8916</td>\n",
       "      <td id=\"T_740b5_row6_col6\" class=\"data row6 col6\" >0.8751</td>\n",
       "      <td id=\"T_740b5_row6_col7\" class=\"data row6 col7\" >0.8775</td>\n",
       "      <td id=\"T_740b5_row6_col8\" class=\"data row6 col8\" >0.9271</td>\n",
       "      <td id=\"T_740b5_row6_col9\" class=\"data row6 col9\" >0.6270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_740b5_level0_row7\" class=\"row_heading level0 row7\" >nb</th>\n",
       "      <td id=\"T_740b5_row7_col0\" class=\"data row7 col0\" >Naive Bayes</td>\n",
       "      <td id=\"T_740b5_row7_col1\" class=\"data row7 col1\" >0.9604</td>\n",
       "      <td id=\"T_740b5_row7_col2\" class=\"data row7 col2\" >0.9638</td>\n",
       "      <td id=\"T_740b5_row7_col3\" class=\"data row7 col3\" >0.8061</td>\n",
       "      <td id=\"T_740b5_row7_col4\" class=\"data row7 col4\" >0.9012</td>\n",
       "      <td id=\"T_740b5_row7_col5\" class=\"data row7 col5\" >0.8479</td>\n",
       "      <td id=\"T_740b5_row7_col6\" class=\"data row7 col6\" >0.8253</td>\n",
       "      <td id=\"T_740b5_row7_col7\" class=\"data row7 col7\" >0.8289</td>\n",
       "      <td id=\"T_740b5_row7_col8\" class=\"data row7 col8\" >0.8890</td>\n",
       "      <td id=\"T_740b5_row7_col9\" class=\"data row7 col9\" >0.0450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_740b5_level0_row8\" class=\"row_heading level0 row8\" >qda</th>\n",
       "      <td id=\"T_740b5_row8_col0\" class=\"data row8 col0\" >Quadratic Discriminant Analysis</td>\n",
       "      <td id=\"T_740b5_row8_col1\" class=\"data row8 col1\" >0.9540</td>\n",
       "      <td id=\"T_740b5_row8_col2\" class=\"data row8 col2\" >0.9634</td>\n",
       "      <td id=\"T_740b5_row8_col3\" class=\"data row8 col3\" >0.8524</td>\n",
       "      <td id=\"T_740b5_row8_col4\" class=\"data row8 col4\" >0.8286</td>\n",
       "      <td id=\"T_740b5_row8_col5\" class=\"data row8 col5\" >0.8370</td>\n",
       "      <td id=\"T_740b5_row8_col6\" class=\"data row8 col6\" >0.8103</td>\n",
       "      <td id=\"T_740b5_row8_col7\" class=\"data row8 col7\" >0.8127</td>\n",
       "      <td id=\"T_740b5_row8_col8\" class=\"data row8 col8\" >0.8608</td>\n",
       "      <td id=\"T_740b5_row8_col9\" class=\"data row8 col9\" >0.0400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_740b5_level0_row9\" class=\"row_heading level0 row9\" >dt</th>\n",
       "      <td id=\"T_740b5_row9_col0\" class=\"data row9 col0\" >Decision Tree Classifier</td>\n",
       "      <td id=\"T_740b5_row9_col1\" class=\"data row9 col1\" >0.9616</td>\n",
       "      <td id=\"T_740b5_row9_col2\" class=\"data row9 col2\" >0.9252</td>\n",
       "      <td id=\"T_740b5_row9_col3\" class=\"data row9 col3\" >0.8749</td>\n",
       "      <td id=\"T_740b5_row9_col4\" class=\"data row9 col4\" >0.8569</td>\n",
       "      <td id=\"T_740b5_row9_col5\" class=\"data row9 col5\" >0.8633</td>\n",
       "      <td id=\"T_740b5_row9_col6\" class=\"data row9 col6\" >0.8411</td>\n",
       "      <td id=\"T_740b5_row9_col7\" class=\"data row9 col7\" >0.8428</td>\n",
       "      <td id=\"T_740b5_row9_col8\" class=\"data row9 col8\" >0.7694</td>\n",
       "      <td id=\"T_740b5_row9_col9\" class=\"data row9 col9\" >0.0770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_740b5_level0_row10\" class=\"row_heading level0 row10\" >knn</th>\n",
       "      <td id=\"T_740b5_row10_col0\" class=\"data row10 col0\" >K Neighbors Classifier</td>\n",
       "      <td id=\"T_740b5_row10_col1\" class=\"data row10 col1\" >0.8601</td>\n",
       "      <td id=\"T_740b5_row10_col2\" class=\"data row10 col2\" >0.7109</td>\n",
       "      <td id=\"T_740b5_row10_col3\" class=\"data row10 col3\" >0.2591</td>\n",
       "      <td id=\"T_740b5_row10_col4\" class=\"data row10 col4\" >0.4819</td>\n",
       "      <td id=\"T_740b5_row10_col5\" class=\"data row10 col5\" >0.3341</td>\n",
       "      <td id=\"T_740b5_row10_col6\" class=\"data row10 col6\" >0.2655</td>\n",
       "      <td id=\"T_740b5_row10_col7\" class=\"data row10 col7\" >0.2821</td>\n",
       "      <td id=\"T_740b5_row10_col8\" class=\"data row10 col8\" >0.3295</td>\n",
       "      <td id=\"T_740b5_row10_col9\" class=\"data row10 col9\" >0.2540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_740b5_level0_row11\" class=\"row_heading level0 row11\" >dummy</th>\n",
       "      <td id=\"T_740b5_row11_col0\" class=\"data row11 col0\" >Dummy Classifier</td>\n",
       "      <td id=\"T_740b5_row11_col1\" class=\"data row11 col1\" >0.8613</td>\n",
       "      <td id=\"T_740b5_row11_col2\" class=\"data row11 col2\" >0.5000</td>\n",
       "      <td id=\"T_740b5_row11_col3\" class=\"data row11 col3\" >0.0000</td>\n",
       "      <td id=\"T_740b5_row11_col4\" class=\"data row11 col4\" >0.0000</td>\n",
       "      <td id=\"T_740b5_row11_col5\" class=\"data row11 col5\" >0.0000</td>\n",
       "      <td id=\"T_740b5_row11_col6\" class=\"data row11 col6\" >0.0000</td>\n",
       "      <td id=\"T_740b5_row11_col7\" class=\"data row11 col7\" >0.0000</td>\n",
       "      <td id=\"T_740b5_row11_col8\" class=\"data row11 col8\" >0.1387</td>\n",
       "      <td id=\"T_740b5_row11_col9\" class=\"data row11 col9\" >0.0190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_740b5_level0_row12\" class=\"row_heading level0 row12\" >svm</th>\n",
       "      <td id=\"T_740b5_row12_col0\" class=\"data row12 col0\" >SVM - Linear Kernel</td>\n",
       "      <td id=\"T_740b5_row12_col1\" class=\"data row12 col1\" >0.4780</td>\n",
       "      <td id=\"T_740b5_row12_col2\" class=\"data row12 col2\" >0.0000</td>\n",
       "      <td id=\"T_740b5_row12_col3\" class=\"data row12 col3\" >0.4494</td>\n",
       "      <td id=\"T_740b5_row12_col4\" class=\"data row12 col4\" >0.1213</td>\n",
       "      <td id=\"T_740b5_row12_col5\" class=\"data row12 col5\" >0.1909</td>\n",
       "      <td id=\"T_740b5_row12_col6\" class=\"data row12 col6\" >-0.0339</td>\n",
       "      <td id=\"T_740b5_row12_col7\" class=\"data row12 col7\" >-0.0469</td>\n",
       "      <td id=\"T_740b5_row12_col8\" class=\"data row12 col8\" >0.0000</td>\n",
       "      <td id=\"T_740b5_row12_col9\" class=\"data row12 col9\" >0.0740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_740b5_level0_row13\" class=\"row_heading level0 row13\" >ridge</th>\n",
       "      <td id=\"T_740b5_row13_col0\" class=\"data row13 col0\" >Ridge Classifier</td>\n",
       "      <td id=\"T_740b5_row13_col1\" class=\"data row13 col1\" >0.9514</td>\n",
       "      <td id=\"T_740b5_row13_col2\" class=\"data row13 col2\" >0.0000</td>\n",
       "      <td id=\"T_740b5_row13_col3\" class=\"data row13 col3\" >0.6491</td>\n",
       "      <td id=\"T_740b5_row13_col4\" class=\"data row13 col4\" >1.0000</td>\n",
       "      <td id=\"T_740b5_row13_col5\" class=\"data row13 col5\" >0.7847</td>\n",
       "      <td id=\"T_740b5_row13_col6\" class=\"data row13 col6\" >0.7589</td>\n",
       "      <td id=\"T_740b5_row13_col7\" class=\"data row13 col7\" >0.7829</td>\n",
       "      <td id=\"T_740b5_row13_col8\" class=\"data row13 col8\" >0.0000</td>\n",
       "      <td id=\"T_740b5_row13_col9\" class=\"data row13 col9\" >0.0340</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x1a615aa8df0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "clf1 = setup(data = df_train, \n",
    "             target = 'target', \n",
    "             numeric_features=numeric_feature_list,\n",
    "             silent=True,\n",
    "             log_experiment = True,\n",
    "             use_gpu=False,\n",
    "             experiment_name = 'pca',\n",
    "             pca = True, pca_components = 10\n",
    "            )\n",
    "add_metric('apc', 'APC', average_precision_score, target = 'pred_proba')\n",
    "best = compare_models(sort=\"APC\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Selected model"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Keeping only performance increasing data preprocessing techniques"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_2ad72 th {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_2ad72_row0_col0, #T_2ad72_row0_col3, #T_2ad72_row1_col0, #T_2ad72_row1_col1, #T_2ad72_row1_col2, #T_2ad72_row1_col3, #T_2ad72_row1_col4, #T_2ad72_row1_col5, #T_2ad72_row1_col6, #T_2ad72_row1_col7, #T_2ad72_row1_col8, #T_2ad72_row2_col0, #T_2ad72_row2_col1, #T_2ad72_row2_col2, #T_2ad72_row2_col3, #T_2ad72_row2_col4, #T_2ad72_row2_col5, #T_2ad72_row2_col6, #T_2ad72_row2_col7, #T_2ad72_row2_col8, #T_2ad72_row3_col0, #T_2ad72_row3_col1, #T_2ad72_row3_col2, #T_2ad72_row3_col3, #T_2ad72_row3_col4, #T_2ad72_row3_col5, #T_2ad72_row3_col6, #T_2ad72_row3_col7, #T_2ad72_row3_col8, #T_2ad72_row4_col0, #T_2ad72_row4_col1, #T_2ad72_row4_col2, #T_2ad72_row4_col3, #T_2ad72_row4_col4, #T_2ad72_row4_col5, #T_2ad72_row4_col6, #T_2ad72_row4_col7, #T_2ad72_row4_col8, #T_2ad72_row5_col0, #T_2ad72_row5_col1, #T_2ad72_row5_col2, #T_2ad72_row5_col3, #T_2ad72_row5_col4, #T_2ad72_row5_col5, #T_2ad72_row5_col6, #T_2ad72_row5_col7, #T_2ad72_row5_col8, #T_2ad72_row6_col0, #T_2ad72_row6_col1, #T_2ad72_row6_col2, #T_2ad72_row6_col4, #T_2ad72_row6_col5, #T_2ad72_row6_col6, #T_2ad72_row6_col7, #T_2ad72_row6_col8, #T_2ad72_row7_col0, #T_2ad72_row7_col1, #T_2ad72_row7_col2, #T_2ad72_row7_col3, #T_2ad72_row7_col4, #T_2ad72_row7_col5, #T_2ad72_row7_col6, #T_2ad72_row7_col7, #T_2ad72_row7_col8, #T_2ad72_row8_col0, #T_2ad72_row8_col1, #T_2ad72_row8_col2, #T_2ad72_row8_col3, #T_2ad72_row8_col4, #T_2ad72_row8_col5, #T_2ad72_row8_col6, #T_2ad72_row8_col7, #T_2ad72_row8_col8, #T_2ad72_row9_col0, #T_2ad72_row9_col1, #T_2ad72_row9_col2, #T_2ad72_row9_col3, #T_2ad72_row9_col4, #T_2ad72_row9_col5, #T_2ad72_row9_col6, #T_2ad72_row9_col7, #T_2ad72_row9_col8, #T_2ad72_row10_col0, #T_2ad72_row10_col1, #T_2ad72_row10_col2, #T_2ad72_row10_col3, #T_2ad72_row10_col4, #T_2ad72_row10_col5, #T_2ad72_row10_col6, #T_2ad72_row10_col7, #T_2ad72_row10_col8, #T_2ad72_row11_col0, #T_2ad72_row11_col1, #T_2ad72_row11_col2, #T_2ad72_row11_col3, #T_2ad72_row11_col4, #T_2ad72_row11_col5, #T_2ad72_row11_col6, #T_2ad72_row11_col7, #T_2ad72_row11_col8, #T_2ad72_row12_col0, #T_2ad72_row12_col1, #T_2ad72_row12_col2, #T_2ad72_row12_col3, #T_2ad72_row12_col4, #T_2ad72_row12_col5, #T_2ad72_row12_col6, #T_2ad72_row12_col7, #T_2ad72_row12_col8, #T_2ad72_row13_col0, #T_2ad72_row13_col1, #T_2ad72_row13_col2, #T_2ad72_row13_col3, #T_2ad72_row13_col4, #T_2ad72_row13_col5, #T_2ad72_row13_col6, #T_2ad72_row13_col7, #T_2ad72_row13_col8 {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_2ad72_row0_col1, #T_2ad72_row0_col2, #T_2ad72_row0_col4, #T_2ad72_row0_col5, #T_2ad72_row0_col6, #T_2ad72_row0_col7, #T_2ad72_row0_col8, #T_2ad72_row6_col3 {\n",
       "  text-align: left;\n",
       "  background-color: yellow;\n",
       "}\n",
       "#T_2ad72_row0_col9, #T_2ad72_row1_col9, #T_2ad72_row2_col9, #T_2ad72_row3_col9, #T_2ad72_row4_col9, #T_2ad72_row5_col9, #T_2ad72_row6_col9, #T_2ad72_row7_col9, #T_2ad72_row8_col9, #T_2ad72_row9_col9, #T_2ad72_row10_col9, #T_2ad72_row12_col9, #T_2ad72_row13_col9 {\n",
       "  text-align: left;\n",
       "  background-color: lightgrey;\n",
       "}\n",
       "#T_2ad72_row11_col9 {\n",
       "  text-align: left;\n",
       "  background-color: yellow;\n",
       "  background-color: lightgrey;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_2ad72\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_2ad72_level0_col0\" class=\"col_heading level0 col0\" >Model</th>\n",
       "      <th id=\"T_2ad72_level0_col1\" class=\"col_heading level0 col1\" >Accuracy</th>\n",
       "      <th id=\"T_2ad72_level0_col2\" class=\"col_heading level0 col2\" >AUC</th>\n",
       "      <th id=\"T_2ad72_level0_col3\" class=\"col_heading level0 col3\" >Recall</th>\n",
       "      <th id=\"T_2ad72_level0_col4\" class=\"col_heading level0 col4\" >Prec.</th>\n",
       "      <th id=\"T_2ad72_level0_col5\" class=\"col_heading level0 col5\" >F1</th>\n",
       "      <th id=\"T_2ad72_level0_col6\" class=\"col_heading level0 col6\" >Kappa</th>\n",
       "      <th id=\"T_2ad72_level0_col7\" class=\"col_heading level0 col7\" >MCC</th>\n",
       "      <th id=\"T_2ad72_level0_col8\" class=\"col_heading level0 col8\" >APC</th>\n",
       "      <th id=\"T_2ad72_level0_col9\" class=\"col_heading level0 col9\" >TT (Sec)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_2ad72_level0_row0\" class=\"row_heading level0 row0\" >et</th>\n",
       "      <td id=\"T_2ad72_row0_col0\" class=\"data row0 col0\" >Extra Trees Classifier</td>\n",
       "      <td id=\"T_2ad72_row0_col1\" class=\"data row0 col1\" >0.9795</td>\n",
       "      <td id=\"T_2ad72_row0_col2\" class=\"data row0 col2\" >0.9847</td>\n",
       "      <td id=\"T_2ad72_row0_col3\" class=\"data row0 col3\" >0.8781</td>\n",
       "      <td id=\"T_2ad72_row0_col4\" class=\"data row0 col4\" >0.9804</td>\n",
       "      <td id=\"T_2ad72_row0_col5\" class=\"data row0 col5\" >0.9254</td>\n",
       "      <td id=\"T_2ad72_row0_col6\" class=\"data row0 col6\" >0.9136</td>\n",
       "      <td id=\"T_2ad72_row0_col7\" class=\"data row0 col7\" >0.9162</td>\n",
       "      <td id=\"T_2ad72_row0_col8\" class=\"data row0 col8\" >0.9604</td>\n",
       "      <td id=\"T_2ad72_row0_col9\" class=\"data row0 col9\" >1.1300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_2ad72_level0_row1\" class=\"row_heading level0 row1\" >rf</th>\n",
       "      <td id=\"T_2ad72_row1_col0\" class=\"data row1 col0\" >Random Forest Classifier</td>\n",
       "      <td id=\"T_2ad72_row1_col1\" class=\"data row1 col1\" >0.9763</td>\n",
       "      <td id=\"T_2ad72_row1_col2\" class=\"data row1 col2\" >0.9820</td>\n",
       "      <td id=\"T_2ad72_row1_col3\" class=\"data row1 col3\" >0.8737</td>\n",
       "      <td id=\"T_2ad72_row1_col4\" class=\"data row1 col4\" >0.9622</td>\n",
       "      <td id=\"T_2ad72_row1_col5\" class=\"data row1 col5\" >0.9150</td>\n",
       "      <td id=\"T_2ad72_row1_col6\" class=\"data row1 col6\" >0.9013</td>\n",
       "      <td id=\"T_2ad72_row1_col7\" class=\"data row1 col7\" >0.9033</td>\n",
       "      <td id=\"T_2ad72_row1_col8\" class=\"data row1 col8\" >0.9565</td>\n",
       "      <td id=\"T_2ad72_row1_col9\" class=\"data row1 col9\" >3.3150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_2ad72_level0_row2\" class=\"row_heading level0 row2\" >gbc</th>\n",
       "      <td id=\"T_2ad72_row2_col0\" class=\"data row2 col0\" >Gradient Boosting Classifier</td>\n",
       "      <td id=\"T_2ad72_row2_col1\" class=\"data row2 col1\" >0.9700</td>\n",
       "      <td id=\"T_2ad72_row2_col2\" class=\"data row2 col2\" >0.9833</td>\n",
       "      <td id=\"T_2ad72_row2_col3\" class=\"data row2 col3\" >0.8735</td>\n",
       "      <td id=\"T_2ad72_row2_col4\" class=\"data row2 col4\" >0.9221</td>\n",
       "      <td id=\"T_2ad72_row2_col5\" class=\"data row2 col5\" >0.8954</td>\n",
       "      <td id=\"T_2ad72_row2_col6\" class=\"data row2 col6\" >0.8779</td>\n",
       "      <td id=\"T_2ad72_row2_col7\" class=\"data row2 col7\" >0.8795</td>\n",
       "      <td id=\"T_2ad72_row2_col8\" class=\"data row2 col8\" >0.9557</td>\n",
       "      <td id=\"T_2ad72_row2_col9\" class=\"data row2 col9\" >15.5030</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_2ad72_level0_row3\" class=\"row_heading level0 row3\" >lightgbm</th>\n",
       "      <td id=\"T_2ad72_row3_col0\" class=\"data row3 col0\" >Light Gradient Boosting Machine</td>\n",
       "      <td id=\"T_2ad72_row3_col1\" class=\"data row3 col1\" >0.9744</td>\n",
       "      <td id=\"T_2ad72_row3_col2\" class=\"data row3 col2\" >0.9805</td>\n",
       "      <td id=\"T_2ad72_row3_col3\" class=\"data row3 col3\" >0.8605</td>\n",
       "      <td id=\"T_2ad72_row3_col4\" class=\"data row3 col4\" >0.9621</td>\n",
       "      <td id=\"T_2ad72_row3_col5\" class=\"data row3 col5\" >0.9075</td>\n",
       "      <td id=\"T_2ad72_row3_col6\" class=\"data row3 col6\" >0.8928</td>\n",
       "      <td id=\"T_2ad72_row3_col7\" class=\"data row3 col7\" >0.8952</td>\n",
       "      <td id=\"T_2ad72_row3_col8\" class=\"data row3 col8\" >0.9538</td>\n",
       "      <td id=\"T_2ad72_row3_col9\" class=\"data row3 col9\" >2.4810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_2ad72_level0_row4\" class=\"row_heading level0 row4\" >ada</th>\n",
       "      <td id=\"T_2ad72_row4_col0\" class=\"data row4 col0\" >Ada Boost Classifier</td>\n",
       "      <td id=\"T_2ad72_row4_col1\" class=\"data row4 col1\" >0.9700</td>\n",
       "      <td id=\"T_2ad72_row4_col2\" class=\"data row4 col2\" >0.9731</td>\n",
       "      <td id=\"T_2ad72_row4_col3\" class=\"data row4 col3\" >0.8866</td>\n",
       "      <td id=\"T_2ad72_row4_col4\" class=\"data row4 col4\" >0.9091</td>\n",
       "      <td id=\"T_2ad72_row4_col5\" class=\"data row4 col5\" >0.8965</td>\n",
       "      <td id=\"T_2ad72_row4_col6\" class=\"data row4 col6\" >0.8790</td>\n",
       "      <td id=\"T_2ad72_row4_col7\" class=\"data row4 col7\" >0.8799</td>\n",
       "      <td id=\"T_2ad72_row4_col8\" class=\"data row4 col8\" >0.9444</td>\n",
       "      <td id=\"T_2ad72_row4_col9\" class=\"data row4 col9\" >3.7310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_2ad72_level0_row5\" class=\"row_heading level0 row5\" >lr</th>\n",
       "      <td id=\"T_2ad72_row5_col0\" class=\"data row5 col0\" >Logistic Regression</td>\n",
       "      <td id=\"T_2ad72_row5_col1\" class=\"data row5 col1\" >0.9553</td>\n",
       "      <td id=\"T_2ad72_row5_col2\" class=\"data row5 col2\" >0.9726</td>\n",
       "      <td id=\"T_2ad72_row5_col3\" class=\"data row5 col3\" >0.9128</td>\n",
       "      <td id=\"T_2ad72_row5_col4\" class=\"data row5 col4\" >0.8134</td>\n",
       "      <td id=\"T_2ad72_row5_col5\" class=\"data row5 col5\" >0.8581</td>\n",
       "      <td id=\"T_2ad72_row5_col6\" class=\"data row5 col6\" >0.8317</td>\n",
       "      <td id=\"T_2ad72_row5_col7\" class=\"data row5 col7\" >0.8352</td>\n",
       "      <td id=\"T_2ad72_row5_col8\" class=\"data row5 col8\" >0.9372</td>\n",
       "      <td id=\"T_2ad72_row5_col9\" class=\"data row5 col9\" >5.0310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_2ad72_level0_row6\" class=\"row_heading level0 row6\" >knn</th>\n",
       "      <td id=\"T_2ad72_row6_col0\" class=\"data row6 col0\" >K Neighbors Classifier</td>\n",
       "      <td id=\"T_2ad72_row6_col1\" class=\"data row6 col1\" >0.9540</td>\n",
       "      <td id=\"T_2ad72_row6_col2\" class=\"data row6 col2\" >0.9683</td>\n",
       "      <td id=\"T_2ad72_row6_col3\" class=\"data row6 col3\" >0.9174</td>\n",
       "      <td id=\"T_2ad72_row6_col4\" class=\"data row6 col4\" >0.8044</td>\n",
       "      <td id=\"T_2ad72_row6_col5\" class=\"data row6 col5\" >0.8552</td>\n",
       "      <td id=\"T_2ad72_row6_col6\" class=\"data row6 col6\" >0.8281</td>\n",
       "      <td id=\"T_2ad72_row6_col7\" class=\"data row6 col7\" >0.8320</td>\n",
       "      <td id=\"T_2ad72_row6_col8\" class=\"data row6 col8\" >0.8982</td>\n",
       "      <td id=\"T_2ad72_row6_col9\" class=\"data row6 col9\" >0.3210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_2ad72_level0_row7\" class=\"row_heading level0 row7\" >lda</th>\n",
       "      <td id=\"T_2ad72_row7_col0\" class=\"data row7 col0\" >Linear Discriminant Analysis</td>\n",
       "      <td id=\"T_2ad72_row7_col1\" class=\"data row7 col1\" >0.9636</td>\n",
       "      <td id=\"T_2ad72_row7_col2\" class=\"data row7 col2\" >0.9709</td>\n",
       "      <td id=\"T_2ad72_row7_col3\" class=\"data row7 col3\" >0.8779</td>\n",
       "      <td id=\"T_2ad72_row7_col4\" class=\"data row7 col4\" >0.8752</td>\n",
       "      <td id=\"T_2ad72_row7_col5\" class=\"data row7 col5\" >0.8755</td>\n",
       "      <td id=\"T_2ad72_row7_col6\" class=\"data row7 col6\" >0.8542</td>\n",
       "      <td id=\"T_2ad72_row7_col7\" class=\"data row7 col7\" >0.8549</td>\n",
       "      <td id=\"T_2ad72_row7_col8\" class=\"data row7 col8\" >0.8962</td>\n",
       "      <td id=\"T_2ad72_row7_col9\" class=\"data row7 col9\" >0.2700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_2ad72_level0_row8\" class=\"row_heading level0 row8\" >nb</th>\n",
       "      <td id=\"T_2ad72_row8_col0\" class=\"data row8 col0\" >Naive Bayes</td>\n",
       "      <td id=\"T_2ad72_row8_col1\" class=\"data row8 col1\" >0.9470</td>\n",
       "      <td id=\"T_2ad72_row8_col2\" class=\"data row8 col2\" >0.9547</td>\n",
       "      <td id=\"T_2ad72_row8_col3\" class=\"data row8 col3\" >0.6682</td>\n",
       "      <td id=\"T_2ad72_row8_col4\" class=\"data row8 col4\" >0.9586</td>\n",
       "      <td id=\"T_2ad72_row8_col5\" class=\"data row8 col5\" >0.7838</td>\n",
       "      <td id=\"T_2ad72_row8_col6\" class=\"data row8 col6\" >0.7550</td>\n",
       "      <td id=\"T_2ad72_row8_col7\" class=\"data row8 col7\" >0.7730</td>\n",
       "      <td id=\"T_2ad72_row8_col8\" class=\"data row8 col8\" >0.8940</td>\n",
       "      <td id=\"T_2ad72_row8_col9\" class=\"data row8 col9\" >0.1110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_2ad72_level0_row9\" class=\"row_heading level0 row9\" >dt</th>\n",
       "      <td id=\"T_2ad72_row9_col0\" class=\"data row9 col0\" >Decision Tree Classifier</td>\n",
       "      <td id=\"T_2ad72_row9_col1\" class=\"data row9 col1\" >0.9457</td>\n",
       "      <td id=\"T_2ad72_row9_col2\" class=\"data row9 col2\" >0.9194</td>\n",
       "      <td id=\"T_2ad72_row9_col3\" class=\"data row9 col3\" >0.8822</td>\n",
       "      <td id=\"T_2ad72_row9_col4\" class=\"data row9 col4\" >0.7847</td>\n",
       "      <td id=\"T_2ad72_row9_col5\" class=\"data row9 col5\" >0.8276</td>\n",
       "      <td id=\"T_2ad72_row9_col6\" class=\"data row9 col6\" >0.7957</td>\n",
       "      <td id=\"T_2ad72_row9_col7\" class=\"data row9 col7\" >0.7997</td>\n",
       "      <td id=\"T_2ad72_row9_col8\" class=\"data row9 col8\" >0.7087</td>\n",
       "      <td id=\"T_2ad72_row9_col9\" class=\"data row9 col9\" >0.4230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_2ad72_level0_row10\" class=\"row_heading level0 row10\" >qda</th>\n",
       "      <td id=\"T_2ad72_row10_col0\" class=\"data row10 col0\" >Quadratic Discriminant Analysis</td>\n",
       "      <td id=\"T_2ad72_row10_col1\" class=\"data row10 col1\" >0.9297</td>\n",
       "      <td id=\"T_2ad72_row10_col2\" class=\"data row10 col2\" >0.8268</td>\n",
       "      <td id=\"T_2ad72_row10_col3\" class=\"data row10 col3\" >0.6812</td>\n",
       "      <td id=\"T_2ad72_row10_col4\" class=\"data row10 col4\" >0.8151</td>\n",
       "      <td id=\"T_2ad72_row10_col5\" class=\"data row10 col5\" >0.7379</td>\n",
       "      <td id=\"T_2ad72_row10_col6\" class=\"data row10 col6\" >0.6979</td>\n",
       "      <td id=\"T_2ad72_row10_col7\" class=\"data row10 col7\" >0.7043</td>\n",
       "      <td id=\"T_2ad72_row10_col8\" class=\"data row10 col8\" >0.6010</td>\n",
       "      <td id=\"T_2ad72_row10_col9\" class=\"data row10 col9\" >0.3610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_2ad72_level0_row11\" class=\"row_heading level0 row11\" >dummy</th>\n",
       "      <td id=\"T_2ad72_row11_col0\" class=\"data row11 col0\" >Dummy Classifier</td>\n",
       "      <td id=\"T_2ad72_row11_col1\" class=\"data row11 col1\" >0.8537</td>\n",
       "      <td id=\"T_2ad72_row11_col2\" class=\"data row11 col2\" >0.5000</td>\n",
       "      <td id=\"T_2ad72_row11_col3\" class=\"data row11 col3\" >0.0000</td>\n",
       "      <td id=\"T_2ad72_row11_col4\" class=\"data row11 col4\" >0.0000</td>\n",
       "      <td id=\"T_2ad72_row11_col5\" class=\"data row11 col5\" >0.0000</td>\n",
       "      <td id=\"T_2ad72_row11_col6\" class=\"data row11 col6\" >0.0000</td>\n",
       "      <td id=\"T_2ad72_row11_col7\" class=\"data row11 col7\" >0.0000</td>\n",
       "      <td id=\"T_2ad72_row11_col8\" class=\"data row11 col8\" >0.1463</td>\n",
       "      <td id=\"T_2ad72_row11_col9\" class=\"data row11 col9\" >0.1090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_2ad72_level0_row12\" class=\"row_heading level0 row12\" >svm</th>\n",
       "      <td id=\"T_2ad72_row12_col0\" class=\"data row12 col0\" >SVM - Linear Kernel</td>\n",
       "      <td id=\"T_2ad72_row12_col1\" class=\"data row12 col1\" >0.9220</td>\n",
       "      <td id=\"T_2ad72_row12_col2\" class=\"data row12 col2\" >0.0000</td>\n",
       "      <td id=\"T_2ad72_row12_col3\" class=\"data row12 col3\" >0.9126</td>\n",
       "      <td id=\"T_2ad72_row12_col4\" class=\"data row12 col4\" >0.6876</td>\n",
       "      <td id=\"T_2ad72_row12_col5\" class=\"data row12 col5\" >0.7789</td>\n",
       "      <td id=\"T_2ad72_row12_col6\" class=\"data row12 col6\" >0.7336</td>\n",
       "      <td id=\"T_2ad72_row12_col7\" class=\"data row12 col7\" >0.7481</td>\n",
       "      <td id=\"T_2ad72_row12_col8\" class=\"data row12 col8\" >0.0000</td>\n",
       "      <td id=\"T_2ad72_row12_col9\" class=\"data row12 col9\" >0.1410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_2ad72_level0_row13\" class=\"row_heading level0 row13\" >ridge</th>\n",
       "      <td id=\"T_2ad72_row13_col0\" class=\"data row13 col0\" >Ridge Classifier</td>\n",
       "      <td id=\"T_2ad72_row13_col1\" class=\"data row13 col1\" >0.9642</td>\n",
       "      <td id=\"T_2ad72_row13_col2\" class=\"data row13 col2\" >0.0000</td>\n",
       "      <td id=\"T_2ad72_row13_col3\" class=\"data row13 col3\" >0.8779</td>\n",
       "      <td id=\"T_2ad72_row13_col4\" class=\"data row13 col4\" >0.8796</td>\n",
       "      <td id=\"T_2ad72_row13_col5\" class=\"data row13 col5\" >0.8777</td>\n",
       "      <td id=\"T_2ad72_row13_col6\" class=\"data row13 col6\" >0.8568</td>\n",
       "      <td id=\"T_2ad72_row13_col7\" class=\"data row13 col7\" >0.8575</td>\n",
       "      <td id=\"T_2ad72_row13_col8\" class=\"data row13 col8\" >0.0000</td>\n",
       "      <td id=\"T_2ad72_row13_col9\" class=\"data row13 col9\" >0.1210</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x1a619cdbaf0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "clf1 = setup(data = df_train, \n",
    "             target = 'target',\n",
    "             numeric_features=numeric_feature_list,\n",
    "             silent=True,\n",
    "             log_experiment = True,\n",
    "             use_gpu=False,\n",
    "             experiment_name = 'selected_model',\n",
    "             feature_interaction = True, feature_ratio = True, \n",
    "             fix_imbalance = True\n",
    "            )\n",
    "add_metric('apc', 'APC', average_precision_score, target = 'pred_proba')\n",
    "best = compare_models(sort=\"APC\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}