10_random_forest.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 51190 entries, 0 to 51189\n",
      "Data columns (total 53 columns):\n",
      " #   Column                         Non-Null Count  Dtype  \n",
      "---  ------                         --------------  -----  \n",
      " 0   age                            51190 non-null  float64\n",
      " 1   job_blue-collar                51190 non-null  float64\n",
      " 2   job_services                   51190 non-null  float64\n",
      " 3   job_entrepreneur               51190 non-null  float64\n",
      " 4   job_admin.                     51190 non-null  float64\n",
      " 5   job_management                 51190 non-null  float64\n",
      " 6   job_technician                 51190 non-null  float64\n",
      " 7   job_housemaid                  51190 non-null  float64\n",
      " 8   job_unemployed                 51190 non-null  float64\n",
      " 9   job_retired                    51190 non-null  float64\n",
      " 10  job_student                    51190 non-null  float64\n",
      " 11  job_self-employed              51190 non-null  float64\n",
      " 12  marital_married                51190 non-null  float64\n",
      " 13  marital_single                 51190 non-null  float64\n",
      " 14  marital_divorced               51190 non-null  float64\n",
      " 15  education_basic.6y             51190 non-null  float64\n",
      " 16  education_university.degree    51190 non-null  float64\n",
      " 17  education_high.school          51190 non-null  float64\n",
      " 18  education_professional.course  51190 non-null  float64\n",
      " 19  education_unknown              51190 non-null  float64\n",
      " 20  education_basic.4y             51190 non-null  float64\n",
      " 21  education_basic.9y             51190 non-null  float64\n",
      " 22  default                        51190 non-null  float64\n",
      " 23  housing                        51190 non-null  float64\n",
      " 24  loan_no                        51190 non-null  float64\n",
      " 25  loan_unknown                   51190 non-null  float64\n",
      " 26  loan_yes                       51190 non-null  float64\n",
      " 27  contact                        51190 non-null  float64\n",
      " 28  month_may                      51190 non-null  float64\n",
      " 29  month_nov                      51190 non-null  float64\n",
      " 30  month_jul                      51190 non-null  float64\n",
      " 31  month_aug                      51190 non-null  float64\n",
      " 32  month_jun                      51190 non-null  float64\n",
      " 33  month_apr                      51190 non-null  float64\n",
      " 34  month_mar                      51190 non-null  float64\n",
      " 35  month_sep                      51190 non-null  float64\n",
      " 36  month_dec                      51190 non-null  float64\n",
      " 37  month_oct                      51190 non-null  float64\n",
      " 38  day_of_week_thu                51190 non-null  float64\n",
      " 39  day_of_week_fri                51190 non-null  float64\n",
      " 40  day_of_week_mon                51190 non-null  float64\n",
      " 41  day_of_week_tue                51190 non-null  float64\n",
      " 42  day_of_week_wed                51190 non-null  float64\n",
      " 43  campaign                       51190 non-null  float64\n",
      " 44  contacted_before               51190 non-null  float64\n",
      " 45  previous                       51190 non-null  float64\n",
      " 46  poutcome_nonexistent           51190 non-null  float64\n",
      " 47  poutcome_failure               51190 non-null  float64\n",
      " 48  poutcome_success               51190 non-null  float64\n",
      " 49  emp.var.rate                   51190 non-null  float64\n",
      " 50  cons.price.idx                 51190 non-null  float64\n",
      " 51  cons.conf.idx                  51190 non-null  float64\n",
      " 52  nr.employed                    51190 non-null  float64\n",
      "dtypes: float64(53)\n",
      "memory usage: 20.7 MB\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import math\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV\n",
    "from sklearn.metrics import classification_report, confusion_matrix, roc_auc_score, auc, f1_score, accuracy_score, precision_score, recall_score, roc_curve, precision_recall_curve \n",
    "from imblearn.over_sampling import SMOTE\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "data = pd.read_csv('data/model_data.csv')\n",
    "\n",
    "X_train = pd.read_csv('data/X_train_transformed.csv')\n",
    "X_test = pd.read_csv('data/X_test_transformed.csv')\n",
    "y_train = pd.read_csv('data/y_train_transformed.csv')\n",
    "y_test = pd.read_csv('data/y_test_transformed.csv')\n",
    "\n",
    "#this step is necessary because we are using the already splot data inside the pycaret setup, and indexes must not overlap on the train and test data sets\n",
    "\n",
    "#combine balanced train data back into a single df\n",
    "X_train['subscribed'] = y_train\n",
    "train_data = X_train\n",
    "\n",
    "#combine test data back into a single df\n",
    "X_test['subscribed'] = y_test\n",
    "test_data = X_test.set_index( np.arange(len(X_train), len(X_train)+len(X_test)))\n",
    "\n",
    "X=train_data.drop(labels=['subscribed'], axis=1)\n",
    "y=train_data['subscribed']\n",
    "\n",
    "X.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_6668d_row8_col1 {\n",
       "  background-color: lightgreen;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_6668d_\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th class=\"col_heading level0 col0\" >Description</th>\n",
       "      <th class=\"col_heading level0 col1\" >Value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_6668d_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "      <td id=\"T_6668d_row0_col0\" class=\"data row0 col0\" >Session id</td>\n",
       "      <td id=\"T_6668d_row0_col1\" class=\"data row0 col1\" >420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6668d_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
       "      <td id=\"T_6668d_row1_col0\" class=\"data row1 col0\" >Target</td>\n",
       "      <td id=\"T_6668d_row1_col1\" class=\"data row1 col1\" >subscribed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6668d_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
       "      <td id=\"T_6668d_row2_col0\" class=\"data row2 col0\" >Target type</td>\n",
       "      <td id=\"T_6668d_row2_col1\" class=\"data row2 col1\" >Binary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6668d_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
       "      <td id=\"T_6668d_row3_col0\" class=\"data row3 col0\" >Original data shape</td>\n",
       "      <td id=\"T_6668d_row3_col1\" class=\"data row3 col1\" >(63546, 54)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6668d_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
       "      <td id=\"T_6668d_row4_col0\" class=\"data row4 col0\" >Transformed data shape</td>\n",
       "      <td id=\"T_6668d_row4_col1\" class=\"data row4 col1\" >(63546, 54)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6668d_level0_row5\" class=\"row_heading level0 row5\" >5</th>\n",
       "      <td id=\"T_6668d_row5_col0\" class=\"data row5 col0\" >Transformed train set shape</td>\n",
       "      <td id=\"T_6668d_row5_col1\" class=\"data row5 col1\" >(51190, 54)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6668d_level0_row6\" class=\"row_heading level0 row6\" >6</th>\n",
       "      <td id=\"T_6668d_row6_col0\" class=\"data row6 col0\" >Transformed test set shape</td>\n",
       "      <td id=\"T_6668d_row6_col1\" class=\"data row6 col1\" >(12356, 54)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6668d_level0_row7\" class=\"row_heading level0 row7\" >7</th>\n",
       "      <td id=\"T_6668d_row7_col0\" class=\"data row7 col0\" >Numeric features</td>\n",
       "      <td id=\"T_6668d_row7_col1\" class=\"data row7 col1\" >53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6668d_level0_row8\" class=\"row_heading level0 row8\" >8</th>\n",
       "      <td id=\"T_6668d_row8_col0\" class=\"data row8 col0\" >Preprocess</td>\n",
       "      <td id=\"T_6668d_row8_col1\" class=\"data row8 col1\" >True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6668d_level0_row9\" class=\"row_heading level0 row9\" >9</th>\n",
       "      <td id=\"T_6668d_row9_col0\" class=\"data row9 col0\" >Imputation type</td>\n",
       "      <td id=\"T_6668d_row9_col1\" class=\"data row9 col1\" >simple</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6668d_level0_row10\" class=\"row_heading level0 row10\" >10</th>\n",
       "      <td id=\"T_6668d_row10_col0\" class=\"data row10 col0\" >Numeric imputation</td>\n",
       "      <td id=\"T_6668d_row10_col1\" class=\"data row10 col1\" >mean</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6668d_level0_row11\" class=\"row_heading level0 row11\" >11</th>\n",
       "      <td id=\"T_6668d_row11_col0\" class=\"data row11 col0\" >Categorical imputation</td>\n",
       "      <td id=\"T_6668d_row11_col1\" class=\"data row11 col1\" >mode</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6668d_level0_row12\" class=\"row_heading level0 row12\" >12</th>\n",
       "      <td id=\"T_6668d_row12_col0\" class=\"data row12 col0\" >Fold Generator</td>\n",
       "      <td id=\"T_6668d_row12_col1\" class=\"data row12 col1\" >StratifiedKFold</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6668d_level0_row13\" class=\"row_heading level0 row13\" >13</th>\n",
       "      <td id=\"T_6668d_row13_col0\" class=\"data row13 col0\" >Fold Number</td>\n",
       "      <td id=\"T_6668d_row13_col1\" class=\"data row13 col1\" >10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6668d_level0_row14\" class=\"row_heading level0 row14\" >14</th>\n",
       "      <td id=\"T_6668d_row14_col0\" class=\"data row14 col0\" >CPU Jobs</td>\n",
       "      <td id=\"T_6668d_row14_col1\" class=\"data row14 col1\" >-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6668d_level0_row15\" class=\"row_heading level0 row15\" >15</th>\n",
       "      <td id=\"T_6668d_row15_col0\" class=\"data row15 col0\" >Use GPU</td>\n",
       "      <td id=\"T_6668d_row15_col1\" class=\"data row15 col1\" >False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6668d_level0_row16\" class=\"row_heading level0 row16\" >16</th>\n",
       "      <td id=\"T_6668d_row16_col0\" class=\"data row16 col0\" >Log Experiment</td>\n",
       "      <td id=\"T_6668d_row16_col1\" class=\"data row16 col1\" >False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6668d_level0_row17\" class=\"row_heading level0 row17\" >17</th>\n",
       "      <td id=\"T_6668d_row17_col0\" class=\"data row17 col0\" >Experiment Name</td>\n",
       "      <td id=\"T_6668d_row17_col1\" class=\"data row17 col1\" >clf-default-name</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6668d_level0_row18\" class=\"row_heading level0 row18\" >18</th>\n",
       "      <td id=\"T_6668d_row18_col0\" class=\"data row18 col0\" >USI</td>\n",
       "      <td id=\"T_6668d_row18_col1\" class=\"data row18 col1\" >7fd3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x29738ff6d60>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pycaret.classification import *\n",
    "s = (setup( data=train_data,\n",
    "            target = 'subscribed',\n",
    "            test_data=test_data,\n",
    "            fold=10,\n",
    "            session_id = 420,\n",
    "            ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "clf = RandomForestClassifier(random_state=420, n_jobs=-1,n_estimators=64, max_depth=3, min_samples_split=0.1, min_samples_leaf=0.05,criterion='gini' )\n",
    "\n",
    "rf = create_model(clf)\n",
    "\n",
    "#tuned_rf = tune_model(rf, n_iter=5, custom_grid = parameters)\n",
    "#calibrated_rf = calibrate_model(rf)\n",
    "#evaluate_model(rf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_model(rf,'saved_rf_model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Transformation Pipeline and Model Successfully Loaded\n"
     ]
    }
   ],
   "source": [
    "\n",
    "rf = load_model('saved_rf_model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2dfcb4334ec149b7849b7b553bc0a560",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(ToggleButtons(description='Plot Type:', icons=('',), options=(('Pipeline Plot', 'pipelin…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluate_model(rf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "</style>\n",
       "<table id=\"T_7bcd2_\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th class=\"col_heading level0 col0\" >Model</th>\n",
       "      <th class=\"col_heading level0 col1\" >Accuracy</th>\n",
       "      <th class=\"col_heading level0 col2\" >AUC</th>\n",
       "      <th class=\"col_heading level0 col3\" >Recall</th>\n",
       "      <th class=\"col_heading level0 col4\" >Prec.</th>\n",
       "      <th class=\"col_heading level0 col5\" >F1</th>\n",
       "      <th class=\"col_heading level0 col6\" >Kappa</th>\n",
       "      <th class=\"col_heading level0 col7\" >MCC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_7bcd2_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "      <td id=\"T_7bcd2_row0_col0\" class=\"data row0 col0\" >Random Forest Classifier</td>\n",
       "      <td id=\"T_7bcd2_row0_col1\" class=\"data row0 col1\" >0.7862</td>\n",
       "      <td id=\"T_7bcd2_row0_col2\" class=\"data row0 col2\" >0.7815</td>\n",
       "      <td id=\"T_7bcd2_row0_col3\" class=\"data row0 col3\" >0.6508</td>\n",
       "      <td id=\"T_7bcd2_row0_col4\" class=\"data row0 col4\" >0.2984</td>\n",
       "      <td id=\"T_7bcd2_row0_col5\" class=\"data row0 col5\" >0.4092</td>\n",
       "      <td id=\"T_7bcd2_row0_col6\" class=\"data row0 col6\" >0.3000</td>\n",
       "      <td id=\"T_7bcd2_row0_col7\" class=\"data row0 col7\" >0.3340</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x2973931ca60>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "rf_holdout_pred = predict_model(rf, probability_threshold=0.5)\n",
    "\n",
    "#plot_model(rf, plot = 'confusion_matrix', plot_kwargs = {'percent' : True})\n",
    "# plot_model(calibrated_rf, plot = 'calibration')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_calibrated_holdout_pred = predict_model(calibrated_rf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_train = predict_model(rf,data=train_data, probability_threshold=0.5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAEFCAYAAACcrEAaAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAc70lEQVR4nO3dd5wV1d3H8c/cbaCwNBFFQVT0Z0lsG2N5JBjLg73FWJIYjVFJgjxqQBIrqIkdTYwllhg0ahI1aozGEmMJdtzYwYOKgAoibSnSdu/M88fM6rIud+7FvTv37nzfec2LuzNn7vwGsj/PmVPGC4IAEZHOLpN0ACIiHUHJTkRSQclORFJByU5EUqEy6QAA6uvra4CdgdlANuFwRDqjCmBDYFJdXd3Ktf2S+vr63kBtnsUX19XVLVjba7W3kkh2hIluYtJBiKTAEODZtTmxvr6+d4W3dH426JbvKQvr6+sHl0rCK5VkNxtg8Hpjqa4oib+XdjN5zu/Ypt/IpMMoihMPHpp0CEUx+pYjufKke5MOo13V9lmHUy47AKLftbX9mmzQjc3XOz/293RVtjfvz7uwF2EtsCR+qUsl2WUBqisWUFM5N+lY2l1nvCeAhk+XJh1C0XTie/vKj4kqMvOoqMj9/+mKwP+ql2l3pZLsRKRM+AT45J6MEHc8CUp2IlIQP/pfXJlSo2QnIgVpCgK8mGZqUwlOQ1WyE5GCZAnIxDRTs2rGiki5C/J4Zhco2YlIucsGAV5MMzWrZqyIlDs/2uLKlBolOxEpiE8Q+0xOQ09EpOw1An5MLivFCe5KdiJSEB8P8PIoU1qU7ESkIH4Aca3UuJpfEpTsRKQgWTwC1exEpLPz80h2cceToGQnIgVpDDy8IPci50GgZCciZc4nQ/wbHTIl984HJTsRKYgfeBBXcws8JTsRKW/ZPIaegFdyyaXU4hGREucHGYKYZ3Zxz/SSoGQnIgUJe2Njkp16Y0Wk3DUGFfhBRc4ymZjjSVCyE5GC+Hh5DBpWzU5EylyWTDT8ZM3imrlJULITkYL4gUc2rgNCg4pFpNz5edTsSm+UnZKdiBQoG3hk8xhUXGqU7ESkIE1U0hTkTh1BCaaW0otIREpaFo+sxtmJSGfn59GMzagZKyLlLp8OirjjSVCyE5GCZINM7NCTjObGiki5awoyNMZMB9NCACJS9rJkYjso4o4nQclORAoSBF64gGdMmVKjZCciBVHNTkRSwQ88/JhncnE1vyQo2YlIQXy8aGn23GVyMbMq4DZgEJAFTgaagAmEr+B+CxjhnPPNbCxwYHT8dOfcy2Y2uK2yua5ZenVNESlpjUFFXluMA4BK59zuwIXAr4GrgHOdc0MIF8Q71Mx2AoYCuwDHANdF53+pbNwFlexEpCBBkMGP2eLeUQFMBSrNLAPUAo1AHfBMdPwRYB9gD+Bx51zgnJsZndN3DWVzUjNWRAqSJX49u7hmLrCUsAn7DrAecBDwLedcEB1fAvQgTITzW5zXvN9ro2xOqtmJSEGal2WP22KcATzmnNsS2J7w+V11i+PdgQZgcfS59X6/jX05KdmJSEH8aLpYri2utxZYCCyKPi8AqoBXzWzPaN/+wETgOWCYmWXMbCCQcc7NW0PZnNSMFZGC+EF8zS2PoSdXA7ea2UTCGt3ZwCvAzWZWDUwB7nXOZaMyLxBWzkZE549qXTbugkp2IlKQxqCCRnL3tsb1xjrnlgJHtXFoaBtlxwHjWu2b2lbZXJTsRKQgAV4ebxfToGIRKXPZIH5Qcew7KhKgZCciBWmnZ3YdTslORAoSBHm8JFvr2YlIuWskQ2NMMmsswVFtSnZrYdVKj/FnDOSTmdWs0y3LqRd/BB5c84sBNDV6VFUHnH3DdGp7Z3nwyn/xhylb0nVdnx+fM4utdlrGxT/ZhIVzqwCY82E1W+30GWf/fkbCd5UO9rUGThg5lbOGf5PNtlzM8DFT8LMejY0Zrjr/6zQsqGGjHrfx2zteYNlnYY/iRaN2pKaLz+iL3qCyymfpoiquPG87li9L569P2IyNeweFR6n1URTtXyua83Y94ejolcBJzrn3inW9jvTInX3oum6W3z70Lh++V8N152xMU5PHj345m63rljHx4R58NK2GxfWVzJuxgGv+OZUlCys45/ubc+2jUz9PbEsaKhhz5GCGX/BxwneUDt/54QfsdeAsViwPk9gpo9/hxsu3ZtrUWvY74kOOPP4Dbrl6K9ap+oDTTq1jccMXA/q/P3wK/36oP08+vBHfO+U9hh32EQ/cNSihO0lWkMcMiVLsjS1mXfMwoItzbjfgl8D4Il6rQ818tws777UEgAGDV/Lem+vQMK+SF/9Vy5nfGcyUV9bFdlzGzKld2GK3TclkoEefLJmKgAWffvHflz9duQGHnjiXPv2akrqVVJn9UVd+PXqHz3++7OztmTa1FoCKioBVqzJ4XkBN5WxGnvM2V/zhJfY95CMAbh6/FU/9sz+eF9C33wqWLqlK4hZKQjZ6lWLcVmqKmez2AB4FcM69CHyjiNfqUJtvu5yX/lVLEMCU+nVYtKCSGa4rOw5ZyuX3vsfSRRU8cXdvNt92OVNf+ICmRpg9o5oZrgsrloV/5Q3zKnn12W7se/SChO8mPZ5/cgOamr74v/zCeTUAbL3dQg4+eiYP3DmILl2zzP1sGFeetx3nj6zjwO9+yKDBSwCPTEXA9Xc/x3bfWMDrk3ondBfJi1vxxM9vuliHK+ZDh1q+mPsGkDWzSufcGqsxk+f8rojhtJ/1h/i88uqTDD/gEwZtvxEbbz2DuTMXEAy6hddmQd+dHM8/P53DfjmMTXd8np8dfCD9NluPDW0B01eMYs7HXXjhnv+y1d4reOOTC5K+nbX2qweTjqBw1RVzGdB7Fr968AQAenZ9gQ2638+0+b9mzO39AJ+5S1dy3t1dAait7cqZNwxk4fIhAMznRLrXvMnv7ruP9+aNTegukhXkMfSkFJuxxUx2rVcryORKdADb9BtJTeXcIobUPia/sg7/u18lu45fzNTXu3Jvw/p0raqh8qMf8/VdPuOld/tTt9Mq+q64ix79RnPTow/z6cdVXHHaQHbf6h8A/OPNQRx72hy22OiahO9m7R1Yt1/SIRRs/Q2X84tL5nLuCRP49v6z2O87H3LCz3dk6eJHABgwaClX/2UmR+2+NV4m4LKbJnHNr7bloKP+wbNP9OONV/pEnRxzOHf4hGRvpgA91+/G6FuObJfvago8mmIahU0pS3bPAQcDd5vZrsCbRbxWh9po05XcdvmG/OWafqxbm+Xn42fSML+S687emGzWY4MBq/jxObMJfJj8zLuc9uAWVNcEjLj4o8+/48P3a9hwk1UJ3kW6ZTIBw8+cwtxPunLOFa8B8NZ/e3PnjYNZsGwI4yc8RLYpw78f7s/Mad148C8DGXHWZI49+X183+P6S7ZJ9gYS5Ocxzi7ueBKKmezuB/Y1s+cJO6F/VMRrdagefbJcdvf7q+3rs0ETV/39y53Nx11+BDtudPeX9t/8tCtafLJmn87uyqgTdgXgmL32brvM0oM594fzV9v30fRunDX8m0WPrxyoGdtK9PKLnxTr+0UkGfkszpnH4p0dLp2jIkVkrfnEvyTb95TsRKTMhe+NVc1ORDq5rO/RFPfCHdXsRKTc6ZmdiKSCmrEikgpBHskuVUNPRKRzUs1ORFIhG2TI+jEdFJl0zaAQkU5IHRQikgpBEP9CnSDooGAKoGQnIgUJAo8gNtmpZiciZS6vDgolOxEpd6rZiUgq+L5H1lcHhYh0cuqNFZFUUDNWRFIhr/XsVLMTkXIXBPHj6DTOTkTKXpjsNKhYRDo534+fG+t7mhsrImVOzVgRSQX1xopIKijZiUgqBNEWV6bUKNmJSEGCwCOImS4WZFSzE5Fyl0czlnJqxprZwFwnOudmtn84IlLq2qs31szOAg4BqoHrgWeACYSt4LeAEc4538zGAgcCTcDpzrmXzWxwW2VzXS/XYJhngKejP1tvT8ffioh0Rs0dFHFbLma2J7A78D/AUGAAcBVwrnNuCOABh5rZTtHxXYBjgOuir/hS2bi411izc85tGneyiKRQQHwzNb5mNwx4E7gfqAXOBE4mrEwBPAL8L+CAx51zATDTzCrNrC9Q10bZ+3NdMPaZnZn1Ai4HNge+C1wB/Nw51xB7OyLS6bRTM3Y9YBPgIGBT4EEgEyU1gCVAD8JEOL/Fec37vTbK5pTPnI6bgUlAn+hLZwN35nGeiHRCzb2xObf4Dor5wGPOuVXOOQesYPWE1R1oABZHn1vv99vYl1M+yW5T59xNgB8Fdg6wcR7niUhnFOS55fYssJ+ZeWbWH1gX+Hf0LA9gf2Ai8BwwzMwyUadpxjk3D3i1jbI55TP0pMnMejSHb2ZbsHpWFZEUaY8ZFM65h8zsW8DLhJWuEcAHwM1mVg1MAe51zmXNbCLwQotyAKNal42LO59kdz5h7+tAM3sA2A04MY/zRKQzaqcpFM65MW3sHtpGuXHAuFb7prZVNpfYZOece8zM6gm7fjPAcOfcnEIuIiKdiRdtcWVKS+wzOzOrAo4lrM19HzjIzErvTkSkY/h5biUmn2bsdYTdvxMI0/XxwHbAacULS0RKVuDlMc6u9OpD+SS7XZ1z2zX/YGYPAa8XLyQRKWXlunhnPkNPPjazzVr83J9wrJ2IpNVXG3aSiFwLATxFGHZf4HUz+w+QBfYgnHgrImnUCZux49awf3wR4hCRMuEF4RZXptTkWgigeZItZrYj0I2wg6KCcC7bM2s4VUQ6M98Lt7gyJSafhQBuI1yKpTfhSOUdCKdw3FrUyESkNJXpuuz5dFB8C9gGuAc4hXBwcXUxgxKREtY+c2M7XD7JbpZzrpGwVredc+5tVl+FQETSpEyTXT7j7D6Olk9+ArjczCB8ficiqZRHb2w5ThcDfgx84JybBNxHOHXsp0WNSkRKVnNvbNxWavJ94c7z0c9/jzYRSasy7aDI1Yx9hjDklvXR5p8DYLO2ThKRzq0zjrPr8BfuHL/LNiycvaijL1tUl06CYf13SDqMosh0aUg6hKLxFzYkHUK78mvacRmS9nnhTofTS7JFpDCdsBkrIvJlSnYikgZeAF5Mq7isntk1W8N7Y0c55xYWOTYRKUVlWrNb2/fG3lHMoESkdJXrODu9N1ZECtO8nl3cVmL03lgRKUyZNmPzSXZj0XtjRSTS6QYVN3POPWpmrxAu7VSB3hsrkm5+fG9sKbb98umNPb/Vrh3MDOfchUWKSURKXQnW3OLk00HhtdiqgUOAfsUMSkRKWGddz845d0HLn83sIuDxokUkIiWtXJ/Z5VOza60bMDC2lIhICcnnmd0HfFEpzQA9CWdRiEgadeKhJ0cBn0afA6DBObe4eCGJSCnz8uiNje2tTUA+ye5259zWRY9ERMpDJ67ZvW5mxwEvA8ubdzrnZhYtKhEpWeXaQZFPstsl2lrSsuwiaVaCySxOrhfuHO+cuy2J5dlFpHSVa80u19CT0zosChEpH36eW4nRSsUiUpByrdnlSnbbmtm0NvZ7QOCc0zM7kTTqhL2x7wEHdFQgIlImOmGyW+Wcm9FhkYhIWeiMzdjnOiwKESkv7ZTMzGx9oB7YF2gCJkTf/hYwwjnnm9lY4MDo+OnOuZfNbHBbZXNda429sc65U7/6rYhIZ9M8XSxui2NmVcCNfDFZ4SrgXOfcEMK+gUPNbCdgKOFY32OA69ZUNu56a7PqiYikWfutZ3cl8HtgVvRzHfBM9PkRYB9gD+Bx51wQzdqqNLO+ayibk5KdiBTMi9nimNkJwFzn3GMtv9Y515wmlwA9gFpgUYsyzfvbKpuTxtmJSGHapzf2RCAws32AHYDbgfVbHO8ONACLo8+t9/tt7MtJNTsRKUh7vCTbOfct59xQ59yewGvAD4FHzGzPqMj+wETCjtJhZpYxs4FAxjk3D3i1jbI5qWYnIoUp3ji7UcDNZlYNTAHudc5lzWwi8AJh5WzEmsrGfbmSnYgUxAvyWLyzgGQX1e6aDW3j+DhgXKt9U9sqm4uSnYgUphPOoBAR+ZLOOINCROTLVLMTkTRQzU5E0iGfxTm1eKeIlDvV7EQkPUowmcVRshORgnhBgBfkznZxx5OgZCcihVFvrIikgZ7ZiUg65LM4p3pjRaTsqRkrImmgZqyIpEcJJrM4SnYiUhDV7EQkFcK3h8WMs1MHhYiUPXVQiEga5PNeWNXsOqlrH5vKsiXhu4vmfFjNk/f14sSzZ5Nt8ujV/a+rla3p6nP1g+9y66835JWna5MIN/UqKn1GXTmNfhutxPc9fnvWplTX+Px03Az8LAzqcwk916ukYV4Vh504m6EHzQdg0tM9ueuajROOvgSoZvdlZrYLcFmrNeY7laoaH88LGHPk4M/3Xfe447IRmzDz3Rr+9sEUBm0VMP2drgCcevFHlOC0wVTZec8GKioCRn13W3bcYxHHj/6QHr2auGHcJkybsi431n+N7w5/mH/c3o9vHzqfMw7fFt+HK++ezPOP92b6O+skfQuJ8sijg6JDIilM0ZKdmY0BjgM+K9Y1SsFm2yynpmvAxX9+n4oK+OOlG/D+W13p3quJyqpqMt4q/Gw1AEf+5FMmT1o34Yjl4w+6UlEZ4HkB63TLkm30uOT/BrNwbvjv5Hk+q1ZmmDu7mvNOMHw//NWtrApoXFmKv8YdLAiI/S92Cf4XvZg1u/eBI4A/5XvCLx78afGiKZIulTMJqt8l02Uvqis/4fIHLmXeZ/uz91EPkPW7sbxxID+76wy61UymV9cX+bDhJAb2vIHN9t6NI1fukHT4qVRVMZ/Nel/JfW4mlZklTJs/hjPv2xKAdaun0nfdG9n28LFcdGjzY4aA/rV3UpHZlRG3n5Rc4CVCz+xacc79zcwGFXLOZYfcwMLZi4oUUXFUVft4GVi1YhIAf5jYwMab38X3dtqG+Z9UcbvbgFduHc3gry1n1caryDS+iFe1ksy8l7nptIFMe7trwnew9jJduiQdwlo5+ZwZvL3KY8IVm7Dehiu59M6LGb3fduy270KOGfEx72cvYcz/3AqE/75nXD6N2UsruO78Qfj+b5INfi313KCWMfec2C7fpXF2KTXsmAUM2moF1569Mb37NVJZBbNnVLP8s7DDojHbi+49slw6YpPPzxl19Uye+XvPsk505WzpokqamsLm6JKGSiorA4YeNJ9hR89lzLHbcPZD/aKSAWNvmsrrL9Ryz439kwu41KgZm06P/rk3o3/zIeMfeBcCj8tOHUjP9Zq45M/TWLXSo1tNJffc0DfpMKWF+2/dgDMum8YVf51MZZXPbeMH8NOx0/l0Vg3n3TCVTde7kB+cvphpk9fh67sspqra5xtDGwD44xUDeOfV7sneQNLyqNmlrjc2DZoaM6vV2po9/2gPAC6dNJqliy5d7dj4MwZ2SGzSthXLKrhk5Bar7Xvq7+t9/vniiadzx29+A8ChW3+zI0MrDxp68mXOuenArsW8hoh0LA09EZF0yAbhFlemxCjZiUhB1BsrIumg3lgRSQPV7EQkHdQbKyJp4PkBXkwHRNzinklQshORgnhBgBfzTC7ueBKU7ESkMGrGikgqqDdWRNJAvbEikg4BedTsOiSSgijZiUhB1BsrIunQDh0UZlYF3AoMAmqAXwGTgQnR2W8BI5xzvpmNBQ4EmoDTnXMvm9ngtsrmumYmJmQRkdVFQ09ybXl0UPwAmO+cGwLsB1wLXAWcG+3zgEPNbCdgKLALcAxwXXT+l8rGXVDJTkQK05zM4rbc7gHOiz57hLW2OuCZaN8jwD7AHsDjzrnAOTcTqDSzvmsom5OasSJSGD/a4srk4JxbCmBm3YF7gXOBK51zzVlyCdADqAXmtzi1eb/XRtmcVLMTkYJ4xDdjvTy6Y81sAPAU8Cfn3F2sniK7Aw3A4uhz6/1tlc1JyU5ECuP7+W05mFk/4HHgF865W6Pdr5rZntHn/YGJwHPAMDPLmNlAIOOcm7eGsjmpGSsihWmHZixwNtALOM/Mmp/dnQZcY2bVwBTgXudc1swmAi8QVs5GRGVHATe3LBt3QSU7ESlIeywE4Jw7jTC5tTa0jbLjgHGt9k1tq2wuSnYiUhjNoBCRVNBCACKSCn4ebxfTdDERKXdavFNE0kHNWBFJBT+Ib6aqGSsiZU81OxFJh3wm+ivZiUi5ywaQjZkiEddbmwAlOxEpTOCHW1yZEqNkJyKF0TM7EUkF9caKSDqog0JE0kDNWBFJhawP2Wx8mRKjZCcihVHNTkRSQclORFIhyKM3VslORMpdEPgEMYOG444nQclORAqj6WIikgp5vCox9ngClOxEpDDqoBCRNAgCnyCm5qZndiJS/lSzE5FU8MljIYAOiaQgSnYiUpAgmyWImS4WdzwJSnYiUqA8Fu8swaqdkp2IFMYPCLSe3VqrAKhdv3vScRRFrw17JB1CUXg1NUmHUDQ9N6hNOoR2Vdu3W/PHinb5rpgOiBbXKxleUAK9JvX19XsAE5OOQyQFhtTV1T27NifW19f3Bt4DeuV5ykJgcF1d3YK1uV57K5Wa3SRgCDAbKL0nmyLlrwLYkPB3ba3U1dUtqK+vHwzkW+1dXCqJDkqkZiciUmyZpAMQEekISnYikgpKdiKSCkp2IpIKSnYikgqlMvSk0zGzDHA9sD2wEjjJOfdeslFJHDPbBbjMObdn0rFI+1LNrngOA7o453YDfgmMTzYciWNmY4BbgC5JxyLtT8muePYAHgVwzr0IfCPZcCQP7wNHJB2EFIeSXfHUAota/Jw1Mz02KGHOub8BjUnHIcWhZFc8i4GWKxtknHNNSQUjknZKdsXzHHAAgJntCryZbDgi6aZmVfHcD+xrZs8DHvCjhOMRSTUtBCAiqaBmrIikgpKdiKSCkp2IpIKSnYikgpKdiKSChp6UCTMbBEwFJgMBUA3MAn7knPtoLb/zBGBP59wJZvZPwsUKZq2h7AXAE865vF+MZGaBc85rtW8cgHNuXI7zpkdxTc/zOrHfKaJkV15mOed2aP7BzC4Bfgcc/lW/2Dl3QEyRocBTX/U6IklRsitv/wEOgc9rQy8BOxC+qW0/4HTCRxX1wAjn3AozOw44l3A62wxgaYvz9wQ+Aa4jXMigEbgIqCFcyOAWMzscWA7cAPQBlgEjnXOvRrXPO4BuwItxwZvZqcBxwLqEr5A/2jk3JTo8zsy2B1YAw51zb5hZP+BGYEBU/izn3BMF/Y1JaumZXZkysyrgaMJpac0ecc4Z0Bc4Gdg9qgl+Cow2s/7A5cC3gN1Yfe5us5GEyWprYB/gfOAvwCuEzdw3gduAMc65nYBTouMA1wIToms+Rw5mVku4DNaezrmvAQ8AP2tR5F3n3I6Eyfa2aN9vgVudc3WESf5GM+ucb1aXdqeaXXnpb2avRZ9rgJcJ18pr9lL057eBLYAXzQzC53v/BXYHnnfOzQEwszuAvVtdYyhwk3POJ6zlbRuVJfqzG7Az8MfmfUA3M+tDWDM8Ntp3J/CHNd2Ic26xmX0POMbMtiSsib7WosgtUbl/mtkdZtaTMPluZWYXRmWqgM3XdA2RlpTsystqz+zasDz6swK42zn3f/B5gqokTGwta/NtrcKy2hJHZjYYmNliVwWwotWzw42BBYQdJ83fHxA2NdtkZgOApwlrg48QJtYdc8S2Krr2Xs65BdF39AfmENYQRXJSM7Zzeho43MzWNzOP8Pna6cCzwK5mtlG0bPzRbZz7H+AoM/PMbH3gGcJaZBNQ6ZxbBLxrZj8AMLN9o3MAngB+EH0+IjpvTXYG3nPOXU1YI92fMJk1+370/YcD7zjnlgFPEjV1zWwb4A1gnbz+RiT1lOw6Iefc68AFhMnhbcJ/50uj5utIwqT0MmEnRWvXA58Br0flRjrnlhCuuvx7M9udMBGdZGZvAJcQdiwEwKnAd6L9BwBLcoT5OJAxs8mEnRnTgU1bHN8yarL/HDg+2jeSMFm/AfwVOC6KTSSWVj0RkVRQzU5EUkHJTkRSQclORFJByU5EUkHJTkRSQclORFJByU5EUuH/AaLXstjiHb1CAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "array([[9697, 1253],\n",
       "       [ 584,  822]], dtype=int64)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix,ConfusionMatrixDisplay\n",
    "import matplotlib.pyplot as plt\n",
    "cm = confusion_matrix(rf_holdout_pred['subscribed'], rf_holdout_pred['prediction_label'])\n",
    "disp = ConfusionMatrixDisplay(confusion_matrix=cm,\n",
    "                              display_labels=rf.classes_)\n",
    "\n",
    "disp.plot()\n",
    "\n",
    "plt.show()\n",
    "\n",
    "cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "interpret_model(rf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "interpret_model(rf, use_train_data=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Threshold=0.538868\n"
     ]
    }
   ],
   "source": [
    "probas = rf.predict_proba(X_test.loc[:, X_test.columns != 'subscribed'])[:,1]\n",
    "probas\n",
    "\n",
    "\n",
    "y_true = rf_holdout_pred['subscribed']\n",
    "y_pred = rf_holdout_pred['prediction_label']\n",
    "predicts = rf_holdout_pred['prediction_score']\n",
    "\n",
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "from numpy import argmax\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(y_true,probas)\n",
    "\n",
    "# get the best threshold\n",
    "J = tpr - fpr\n",
    "ix = argmax(J)\n",
    "best_thresh = thresholds[ix]\n",
    "print('Best Threshold=%f' % (best_thresh))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEFCAYAAADqujDUAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA560lEQVR4nO3deXxU1dnA8d/MZLKRhSXsOyQclrAGSIKAqKCigoiAK9Z9rbXVKq22Fa3Wt4r19bW1KhWtrRuKqKi4g7KFJSiLwGEzLAKBBEgCSSbJzLx/zIBZh0kyM3eW5/v58JF77vbcBM9z1+eYnE4nQgghIo/Z6ACEEEIYQxKAEEJEKEkAQggRoSQBCCFEhIoyOgBv5ObmxgAjgIOA3eBwhBAiVFiAjsDajIwMW+2ZIZEAcHX+y4wOQgghQtQYYHntxlBJAAcB+vTpQ3R0dKNX3rx5M+np6T4PKpjJMUcGOebI0NRjrqioYPv27eDuQ2sLlQRgB4iOjiYmJqZJG2jqeqFMjjkyyDFHhmYec723zuUhsBBCRChJAEIIEaEkAQghRISSBCCEEBFKEoAQQkQov74FpJTKBP6qtR5Xq30S8CegCpintZ7rzziEECLU5BflsWrHQk5WlBDjSCa1rDvJcSk+3YffEoBS6gFgJnCyVrsVeAbXx10ngRVKqQ+11vn+ikUIIYLVrsMbWLn9feyUNbhMJaUszJ3DZRm/9WkS8OcVwC5gKvCfWu39gJ1a62MASqnlwFjgnTNtcPPmzU0OJjc3t8nrNmTLli08/fTTPPnkk7Rp0waAN998k06dOnH22Wefcf3i4mJefvllysvLKS8vp3Pnzlx//fUUFRXx3HPP8eijj55e9vjx47z33nvceOON/OpXv2LOnDnMmzeP7OxsBg8eXO/2/XHMwU6OOTKE2jFXOsr5yZZLCQeatZ3l339Eh+iBPorKjwlAa71AKdWjnllJQFG16RIg2ZttpqenN+ljiNzcXDIyMhq93plUVVURFxfHG2+8wSuvvILJZGLJkiX06NHDq/09+eSTXHzxxVx11VUAPP7442itGT9+PC1atKizjfPOOw9wfRAybNgwPvjgA1JTU+vdl7+OOZjJMUeGUDjmSruN3N2fsS1/pU+3O3rIJY26ArDZbB5PnI34ErgYSKw2nQgc98WGez32Xr3tM3oncurfy3VvLGf57sN1lsnsnsKbM8cCMDdnB098uYndf5h6xn1mZWXhcDh4/fXXufbaa2vMmzdvHh9//DFRUVEMHz6c+++/v8b8lJQUPvvsM7p3786wYcOYNWsWJpOJgwddX23b7XZ+97vfkZaWxkUXXcS9997L/PnzzxiTECKw8ovyWL7tHUoqCwErUOnT7cfQmosybgydZwAebAXSlFKtgRO4bv/MMSAOn5k9ezbTp09nzJgxp9u01ixevJi33nqLqKgo7r77bpYsWcI555xzepnrr7+epKQkXn75Ze655x4yMjJ4+OGHAdfVxW9/+1uGDx/ONddcw/79+wN+XEIIl6KyAlbv/JADRTuAMw2j27TO3+kEux3sToixmogxJ5IQm8TI3pewf2ehzzt/CGACUEpdDSRorV9SSt0LfIbrNdR5WuuffLGPhs7Yq98vfO3q0Wfczi1ZadySleb1flu1asWDDz7IrFmzGDZsmCuW3bsZPHgwVqsVgOHDh7Njx44aCSAnJ4cpU6Ywbdo0KioqmDt3Ln/5y1+YNWsWWmsSEhIoLS31Og4hRPPsOryBZdvf5edO3Pdn87VVVUFBqYVXv+tMVq/BPD15OG1a1LzVvZ9Cv+zbrwlAa50HZLn//ka19kXAIn/uO9DOPfdcvvjiCxYuXMj9999Pr169eOWVV6iqqsJisbB27VqmTJlSY53XXnuNw4cPM2XKFKKjo0lLS2P37t0ADBgwgJdeeun0lUVCQoIBRyVE+Nu8fznr8j5qYK7vOn8zFob1mEh6l59PQl9bt4vb3lxJ15bx/HNaFhP7dfbZ/rwRKtVAQ8JDDz1ETk4OAEopJk6cyFVXXYXD4SAjI4Px48fXWP6RRx7hkUce4dVXXyU2NpZWrVoxe/ZsKitd/+hiY2N5+OGHmTVrFs8880zAj0eIcFFaUUzOjkXsPbbJ3WLizLdymsp11WA1x5CVOpXe7Wq+pXfSVonZbCLOGsW1w3pRcMLGLVlpJMZa/RRPw0xOp79+CL6Tm5vbA/gx2N4CCmZyzJFBjrlhG/Yu5bu9nwYgIjNRWOnQqjcjel3k8V79V9sPcts7OUwd1I0nJ3n/e2vq77naW0A9MzIy8mrPlysAIUTIqntm72s/PwMwYSajx0U1buF463hZBfd/mMu8NTuxmE1EW8w4nU5MJpOP420cSQBCiJCwdven/HBgKQCblp/xu9EmS4xpw+g+02mf3MMn2/tg8z7uWrCag8VlDO7Uirkzssno2sYn224uSQBCiKC0p2ALy7ctoLJmNRkfqfkMoGVce7JTL/NZp3/K1vwipr6ylGiLmT9PHML95wzAagmeGpySAIQQhikqKyB392Lyj++l0lmJg3L89YDWao5hRK9J9Okw3Ofbrs7pdHLCVkVirJV+7ZP53ynDOS+tI/07tPTrfptCEoAQImC8ezDrm87fjIURvSbRr1OWT7bnjb3HTnLHuzlU2h18dtt4TCYTd4/pF7D9N5YkACGEX+Ts/Ihth5b7fT+BOrP3xOFw8uKq7fzu4/WcsFUxvk9HSmyVJMVGGxaTNyQBNNPq1av59a9/TWpqKgAnT56kS5cuzJkzh2HDhjF06NDTy/bu3ZvZs2cbFKkQgbNg9RxKKgt8vt22CT1oVdGPUSPPXG03UPThIm57J4dluw/TMi6al68YxS9G9DL8DR9vRFwCqLTbOF6aT8v49lgtjf+moD5ZWVk1PtS67777+Prrr0lOTuY//6ldDVuI8JNflMfK7Qsosh1pxlbMgANXt1QFWIm3tiCz92S6p/Q/vVQwlYIur7RzzvOfk19SzmUDu/H3qSPpkBRndFheC5sEsPbHT8gr2FjvvIqKCnav/QKn00lZZQlOpwOTyUycNdFjlu6RMogRPS9qVBwVFRUcPnyY5GSvKlwLEVL88XFVUmw7zkqb6vM3cPzJVmUnJspCrNXCnMnDiYkyc/mg7kaH1WhhkwC84XDacTodADidDhxOOxZT838EOTk5zJw5k8LCQsxmMzNmzCA7O5uioiJmzpx5erlZs2aRnp7e7P0JEShFZQWs2P4eh0t2N3NLp97sMdMxuTdZqZf6pbqlv5VX2nn8y43M/34PufdeTEKMlauH9TQ6rCYLmwQwoudFDZ6tn/qMutJu46Pv/05R2RGS49pyyZBf+uQ20KlbQMeOHePGG2+kS5cuAHILSISMSruNHYfWsSnvG8qcxT7d9oyRDxIfneTTbRph5Y+HuWX+KrYdLqZ7qxbkHT1BesdWRofVLGGTALxhtcRwyZBf+vwZwCmtWrXiqaee4rrrruP999/36baF8Iefju5gxdZ3KPVxpx9tjmdIt/NI6zjc5/+fBdoJWyUPffId/1ihAfjlaMXjFw0lISbwxdt8LaISALiSQNvEbn7bfmpqKjNnzuSxxx7z2z6EaKrth9axcudCwO7jLUcTY4lmZO9Jdapfhror/7OMxVt/QrVNYu4V2ZzVs53RIflMxCUAX8vMzCQzM7NG2x133GFQNELUVGm3sXbnJ2w/stpn2+zeejCj1dSQP7P3pMruIMpdsuGPEwYyuFMr/jhhELFWi8GR+ZYkACHCzNYDOaze/b7PtmfCzMhekwP6Ra2R3tu4lwcW5fLJrefRp20Smd3bktm9rdFh+YUkACHCRHNf0Yy0jr62Q8Vl3L1wDe9t3EtMlJn1+wvp0zb0H157IglAiBCSX5TH8m3vUFL58xixTS+NbAbMxFnjyeo9pcbHVpHE6XTy2rrd3PfBOo6VVTC6ZztempGFahf+3/JIAhAiBHget9Z7ZqLp1ymLId3PC+t7+I3x168389An35MQE8Xfp47ktuw+mM3BX8bBFyQBCBGEfNXhnzKg0zhG9LrQZ9sLddVH47phZCobDhzjfy4eRvfWCQZHFliSAIQwWFFZAat2vM+h4p38XA+neWIs8YxKmxaxt3U82ZZfxK3zV3H/uQOYNKAr7RPjeHPmWKPDMoQkACEMsuvwBpZtf7NWa/M6/9bxXTirz1TaJHRq1nbCUaXdwZwlP/Do5xupsDsYog8yaUBXo8MylCQAIQKstKKYLza9xrGy/U3exqAu4xnWY/zpMifCs+/2H+Xmt1fy/YFjdEiM47mpI5k6yH8fhIYKSQBCBIAvvsCNhA+w/GHpzkOc/+KX2B1ObhjZm6cmZdAqXn6GIAlACL9ydfzvNnItMwlRLcECI3peIvfxm+jUg96zerbjkv5duGNUHyYouTVWnSQAIXzMdW//bRpzP9+EhdF9ZoRdHR0jlJRX8vuP19M5OZ7fjx+I1WLmvRvGGR1WUJIEIISPrM/7ko37v2zkWiZGpV5u6Hi24WTx1p+4490c9h0vZUinVtx/zoDTNX1EXZIAhGiG5ryv37fDaLJSL/FxRJGp8KSNez9Yx39zdxNlNvGHCQN5cPxA6fzPQBKAEI3U1Jo7VnM84wdcF1JDH4aCwyVlDJ7zEYdPlJPRpTX/umIUgzqF9kAtgSIJQAgvNKfCZkJMG8b0mS4dv5+0S4xj0oAu9GmbxK/H9pOz/kbwWwJQSpmB54HBgA24WWu9s9r8+4CrcT0p+4vWeqG/YhGisUorilm6dT6HS3aeeeF6xFgSObf/NdLp+4HT6WTe6p2syjvC3CuyAXhpRrbBUYUmf14BTAFitdbZSqks4GngUgClVEvgHiAVaAF8D0gCEIbZU7CFZdvepYrSZm3n1Adawj9+LCzh7iV7WXNoK4kxVv4wYWDE1e/xJX8mgNHApwBa6xylVPXXHE4Ce3B1/i3wRfETIZro263vsLswt0nrmrCQ3mUsg7qOkw+0/MjucPCP5ZqHFn9HaYWdif0688/LM+naqoXRoYU0fyaAJKCo2rRdKRWlta5yT+8DtgAW4AlvNrh58+YmB5Ob27T/wUOZHHPDTlYWsLtqNTTxjN9KIt2tWcRFtYRC2FjY9H+bzRXuv2en08ldX+9hXX4pyTEWHh3VmQu6J3F49zYOGx1cAPnj9+zPBFAMJFabNlfr/CcCHYGe7unPlFIrtNZrPG0wPT2dmJjGn2VFYr0UOeb67SnYwpJtr9OUkgwdkxVj1OXERwfPKFGR8nueURJH7/1HefayEezVP0TEMVfX1N+zzWbzeOLszwSwApgEzHc/A9hUbd4xoAywaa2dSqnjQEs/xiIiXH5RHl9veh0bJY1Yy8zwHheR3mW03+IS9Vu3r5A5S37g31efRUyUhXvH9T9dv3+vwbGFE38mgIXABKXUSsAE3KCUuhfYqbX+UCk1HshRSjmA5cAXfoxFRKjSimJydixi77FNZ14YGN7jEunwDVRWWcXsTzfwt2+24nA6uXJoD6YM7Ha68xe+5bcEoLV2ALfXat5Wbf7DwMP+2r+IPHZnFcu3vcvOgnWNXrdXmwzG9pvuh6iEt77Zlc+t81exs6CEXm0SeHF6FuemdTQ6rLAmH4KJkJaz8yO2HVr+c0O59+tazXGM63s1nVun+T4w0SiPfbGRhz/dgNlk4jdn9+PRC4cQHy3dk7/JT1iEhJrDJjaP1RzDuL7XSscfRDK7pZDeoSUvzcgis3tbo8OJGJIARFBrWj39hpgY3uNiuccfBApOlPPgJ9/xp/MH0aVlCyaoTqy/rwMWs5RxCCRJACLolFYUs3L7++w/vqWZW7IAdkxYyOgxUTr+IOB0Onn7+zzuWbiWgpM2WsZF8+Qk1+uN0vkHniQAYZjGvqHjDRNmMuTVzaD0U1Epdy1YzaIf9hNntfDUpAzuGdvX6LAimiQAEXCFJw7wzdY3KLYVNHtbJqz0ajuQjJ4XsnXTjoj7QChULN76E1f/dxnF5ZWM692eF2dkkZoSPB/URSpJAMKvmlo7vyGjUqfJ6FkhqF/7ZBKio3hyUgY3Z6bKe/1BQhKA8Ivm1M+vzWKyktn7Uun4Q4jd4eDZb7cxvGsbxvZuT4/WCex86DJioixGhyaqkQQgfGrl9g/YfniVD7YkZRhC1eaDx7j57VWs3VfIqB5tWXb3hQDS+QchSQCiySrtNr7L+5ItB1diwoqzMV9hVTO024UM7jbOt8GJgKuosvPEV5t54qvNVNodXD2sJ89cKldtwUwSgGiU/KI8Vm5fQJHtSI12Z6Oqa1qItyaQ2ftSuqf0922AwhC7C0uYMm8JPxwqoktyPM9Py+Ti/l2MDkucwRkTgFKqFfAk0BuYDjwF3Ke1Pubn2EQQKTxxgKVbXqekorDR68ZbW3HegJm0Sejkh8hEMOiQGEdFlYPbsvvwP5cMJSk22uiQhBe8uQKYC3wOjARKgIPAf4GL/RiXCBJrdy3mh4PfNGndpJi2nN3vKun4w9SSnYc4UFTKNRm9iI+OYt29F5MQYzU6LNEI3iSAnlrrl5RSd2itK4CHlFIb/B2YMEZz394xYSEloTNZqVOk4w9TRWUVzPpoPXNzdpAca2XSgC4kxUZL5x+CvEkAVUqpZMAJoJRKQ8bwDUtLt7xF3tHvG7mWiUFdzpOB0CPEoh/2cee7qzlQXMbAji351xWj5HZPCPMmATwMLAW6KaXeB7KBG/0YkzBAYzp/izmGUalT6d1usH+DEkGjvNLOTW+v5K3v8oi2mHn0wsHcf84AouXVzpB2xgSgtf5UKbUOyMRVXes2XEM6ijCwPu9LNu7/0qtl27Toyqi0y+TWTgSKiTJTXF5JVvcU5s7Ipn+HlkaHJHzAm7eAVmmts4GP3dNmYAMw0M+xCT/5+f39ZV4tnxTbjrPSptI+uYd/AxNBZf/xk3y4eT93jlaYTCb+e81oEmKipGpnGGkwASilvgbGuf/uwP0MALADH/o9MuFzlXYb2w7kkLtn8RmXTU0Zyei+UwMQlQg2DoeTuat3MGvRekpslQzv1oaR3VJIjpN7/eGmwQSgtT4XQCn1rNb6nsCFJHytsYOqZPaaQr9OWX6MSASrHUeKue2dHL7ZlU9yrJWXZmQxomsbo8MSfuLNQ+BZSqnLgATAhOs5QE+t9Z/8GploFruzipwdH7Itf6XX63Rtlc7YvtOxWmL8GJkIVs9+u5UHP/6O8io7kwd04R+XZ9IpOd7osIQfeZMAFgDxQCqwDBgL+KLal/CTwhMH2FK+0OsB0tsl9eSstMtJjkvxb2AiqBWetJEUa+XVqWcxbVA3KdkcAbxJAApIA54F5gG/BXw1SKvwsW+3vsPuwlwvlpTxcSOdrcrOvNU7uTU7DYvZzEMTBnLP2H60aSFXgJHCmwSQr7V2KqW2AYO01q8ppeRfSJD56egOvtjyslfLDug0jhG9LvRzRCKYrco7wi3zV7E1vwinE+4crYiJskjJ5gjjTQL4QSn1HPBP4HWlVCdAvvkOAo0ZbSvemsRZadPp3DrNz1GJYHbSVskfFn/Pc8u3uTr+sxQzh/cyOixhEG8SwB3AKK31FqXUw8B5wFX+DUt40piPt5Ki2zF15L1+jkiEgqU7D3HT2yvJO3qStJRE5l6RzZhe7Y0OSxjIYwJQSimgRGu9DEBr/aFSajXwZ+DWAMQncD3UXbFjAUdP/tSo9eRdflFdwUkb+46XMuvcAfzx/EHEWWU4kEjn6UOw2bge+KKUmgIscU8/COQEILaIV1pRzJeb/8vR0r2NWi/a0oJu5lGM7nuenyIToeKDzfsY3bMdbVrEcPmgbmyZNZnUlCSjwxJBwtMpwHW43v7pBDwKzAI6ADO01p8FILaIVlpRzPw1f2nUOmYsDOsxkfQuo8nN9eZNIBGu8kvK+NXCtby7YQ+/GNGbeVeOwmQySecvavCUAEq01geBg0qpkcBrwIVa68aM/ScaqaEhFxuS0qI7Fwy6UT7eEgA4nU7+k7ube99fx7GyCkb1aMsD5wwwOiwRpDwlgOo1/wu01vf5O5hI15iSDfLxlqht37GT3PZuDp9tO0CL6CienTKCO89SmM3yQZeon6cE4Kz29zJ/BxLJKu02Vu9YxM6CdR6Xk/f3hScltkqW7DjE+D4deXF6Fj1aJxgdkghynhLAAKXUbvffO1f7uwlwaq09vjzsLhv9PDAYsAE3a613Vps/EddgMyYgF7hLa+2sb1vhrKisgIW5c8643Dl9r6N7Sv8ARCRCyZ5iG7GHjjOgQ0v6d2jJ6l9fxMCOLaWMg/CKpwTQp5nbngLEaq2zlVJZwNPApQBKqUTgKWCc1rpAKfUAkAJ4d+M7THgz/m68tSWZvSdL5y9qqLI7+Ns3W3h48W76rz/K6l9fRJTFzKBOrYwOTYQQk9Ppn5NupdTfgDVa67fc0z9prTu7/34BcD1QAfQC/qW1/ndD28rNze0B/OiXQA2yq+wbSjnc4HwrLekbNyGAEYlQsf1YOY/lHGDbsXJax1p4YHhHzu0mb/cIj3pmZGTk1W7055cgSUBRtWm7UipKa12F62z/HGAIcAJY5h55bLunDaanpxMT0/i3XXJzc8nIyGj0er6Ws/Mjth1afsblfHGvP1iOOZDC/ZjLK+08/uVGnvz6R6ocTn4xojfXdovi3FEjjQ4toML991yfph6zzWZj8+bNDc73ZwIoBhKrTZvdnT9AIbBWa30IQCn1La5k4DEBhLIP1v0fx8oPeFxGyjYIT2xVdl5ds4vOyfG8MD2L81Un+d5DNItXCUApdRauMYBfATK11t96sdoKYBIw3/0MYFO1eeuBdKVUCnAcyALmNiLukLIgZw4lVQUel+nRegjj+l8ZoIhEqDhhq+SHQ8fJ7N6W5LhoPrrlXHq3SSQhRuoxiubzZlD4e3A90O0MvAO8qJR6WWt9pldXFgITlFIrcb3pc4NS6l5gp7um0O+BU18Uz9daN3ydEsLmr/wrpY5jHpeZ0P8mqdIp6vhcH+D2d3IosVWy+YHJtE+MY3Cn1kaHJcKIN1cA1wOZwGqtdaFSagSwBvCYALTWDuD2Ws3bqs1/C3irUdGGmAWr53js/Nu06MbYvjPkYy5Rw9FSG7/9MJd/r92FxWzigXMGkBwrA7IL3/MmAdi11hWuwqCAa6BBKQfhQVFZAZ/kvoiNknrnTxx4O+2TewQ2KBESFmzcw93vrSG/pJyhnVvzryuyGdJZzvqFf3iTAL5RSs0BWrirgt4KfOXXqELYnoItLNn2WoPzpfMXDXE6nTy3bBvHyyp44uKh/Obs/lgtZqPDEmHMmwRwP3ALsAFXhdBPgBf8GVSoKjxxwGPnf1nGb+V2j6jB6XSybl8hI7qlYDKZePmKUVQ5HKh2yUaHJiKANwngb8B/tdYv+juYUPfFplcanDemz1XS+Ysa8o6e4LZ3cvh6xyGW330Bmd3b0jsl8cwrCuEj3iSAHcD/KqVaA2/gSgZ5fo0qBO0p2EK5ve49f6s5hnF9r5W3fMRpDoeT51doHvzkO05WVHFB3050TIo3OiwRgc6YALTW/wD+oZTqBswA3ldKndBaj/Z7dCHkW/1mnbaWsR2YMvzXgQ9GBK2t+UXcOn8VK/OO0Do+mn9cfhbXZvSU4m3CEN5+CJYMjAfOd68jI4JVs/3QOuzOyjrtFw+9w4BoRDB7cdV2VuYdYfrg7jx72QjaJ8YZHZKIYN58CLYIGAq8B/xRa73a71GFmNW7F9VpS0noIaN0CQC25Reh2iVhMpn484VDOC+tA5MGdDU6LCG8ugJ4CVhcrY6PqMXusNVpG6OmGRCJCCZllVX8+fONzFm6hX9Oy+SmzDQSY63S+Yug0WACUErN1lrPBqYCl1X7EAwArfWN/g0tNOw6vKHednnjJ7It253PrfNz2H6kmJ6tE+gpo3OJIOTpCuBUmcGl9cyLuJG7GrJi+/w6bZ2SmzuWjghVJeWV/P7j9fxz5XZMJrhnbF/+fOEQWkjxNhGEGkwAWutTN7Y7aa2fqD5PKfUXv0YVInYd3oCjnqoYmamTDYhGBIP3N+/jnyu30799MnOvyCare1ujQxKiQZ5uAf0P0A6YrJSq/hJ7FK7yzQ/6Obagl7Pz/Tpt7RJT5fZPhCk8aSPaYiYx1sq1GT2ptDu4JqMnMVEWo0MTwiNPhUYWAN8AJ93/PfXnM+Bi/4cW3ApPHKDSUVanfVy/GQZEI4zgdDp5Z8Me0p/8kIc++Q4Ak8nEjZmp0vmLkODpFtBaYK1SaqHWujiAMYWEZfqdOm0tY9sTHy1js0aCg8Wl3LVgDR9s3kdslIXurVoYHZIQjebpFtB6rfUw4LhSqvpDXxPg1FpH9CnO8bKDddqy0y4zIBIRSE6nk1fW7OK3H66jqLySsb3a8dKMbNLaSuIXocfTFcAw93+lHm0tDb36KWWew9+W/CJueyeHFtFRPD8tk1sy0zCbpYyDCE3efAncG9dD3zdwlYEeBvxGa73cz7EFpUq7jWXb66v7096AaEQg2B0OjpVWkJIQy4AOLZk7I5vz0jrQVW77iBDnzdn9K0AFcCmggHs5w3CQ4eybLW/X2y63f8LTD4eOM+a5z5j2729wOFx3Qq8f2Vs6fxEWvEkAsVrrd4BLgNe11suAiPyqpaisgP1FW+q092mXLbd/wkxFlZ3HvthIxt8+ZvXeAjolxVNWKdVQRHjxakxgpdTluBLAH93DQkbkmMC5P35aT6uVUX0uDXgswn/W7i3glvmr2HTwOJ2S4vjH5ZlMTpf6PSL8eJMAbgV+A9yptT6olLoSuNm/YQWnQ8d/rNM2ceBNBkQi/KWssorJLy/h8Ilybs5K5clLMkiOizY6LCH8wpsBYTYppZ4BzlZK/Rr4H631Rr9HFoQqHCfrtMmtn/BQVFZBclw0cdYo/jktk6RYK+emdTQ6LCH86ozPAJRSM4H3gZ5Ad+A9pVTEVQLdeiCnTpuZWAMiEb5UVFbBHe/mMOipRRSVVQAwZWA36fxFRPDmFtB9wEitdSGAUupxXBVC5/kxrqCztp5BXwZ1HWNAJMJXPt6ynzveXc1PRaUM6JBM/olyud0jIoo3CcByqvMH0FoXKKUcfowp6BSVFdRb9XNAFxkWORQdOVHOb95fy5vf5WG1mHn4/EH87rx0oqV+j4gw3iSADUqp/wVedk/fBNT/KWyYytn5QZ22DkmpMuRjiJr5+nK+2H6Qkd3aMHdGNukdWxkdkhCG8CYB3ALMxnXLxwx8Bdzpx5iCTn5RXp227LQpAY9DNF1ZZRVxVtc/9ycuHsb56hD3jO2LxSyVTkTk8pgAlFIpuB78PqK1nhWYkIJP64TOFJzI+3k6vovU/A8RTqeTf63eyR8Xf8dXd5zPgA4tGdqlNUO7tDY6NCEM1+Dpj1JqOpAHfAzkKaXODlRQweZE2dEa01WOcoMiEY2xq6CECS98we3v5GCrcrCroMTokIQIKp6uAP4AjNBab1VKXQA8AowLSFRBJL8oj3J7zeEQzF5V0BBGsTscPPvtNv706feUVdq5pH8Xnp+WSefkeKNDEyKoeEoATq31VgCt9WdKqUYVgFNKmYHngcGADbhZa72znmU+Bj7QWr/QqMgDZNWuug+Aq+xSEyaYPfbFJh79fCNtE2J4+YpRzBjSHZNJSjYLUZunBFD7Vc/KRm57Cq5CctlKqSzgaVwVRat7DAjqVzCOl+bXaRvR+xIDIhGeVNodOJ2uap2/HN2X/JJyHr1wMCkJ8rGeEA3xlAASlVJjcI0ABpBQfVpr/e0Ztj0a+NS9bI5Sanj1mUqpabiSTH0V1oJC4YkD1M2D0D2lf+CDEQ1as7eAW95exTWpCQwfDm1axPD8tEyjwxIi6HlKAPuBR6tN/1Rt2gmce4ZtJwFF1abtSqkorXWVUioduBqYBvzJ22A3b97s7aJ15ObmNnqdXWVL67TF0KpJ2zJCqMTZVOVVDl7YeJi39FEcTtjeJirsj7k+csyRwR/H7GlIyHOaue1iILHatFlrferm+XVAZ+BroAdQoZTK01p7vBpIT08nJqbxH1/l5uaSkZHR6PW2rfjYleqqOX/ITNokdGr0tgKtqcccKpbsPMSt81exu/AEqSmJvDQjm4Tj+8P6mOsT7r/n+sgxe89ms3k8cfbmQ7CmWgFMAua7nwFsOjVDa/3Aqb8rpWYDh87U+Ruh0llapy0UOv9w97k+wMSXvsJsMnH/OQN4+IJBxFmjyM3db3RoQoQUfyaAhcAEpdRKXM8NblBK3Qvs1Fp/6Mf9+kR9X//698clzsThcGI2mzgvrQO/GNGbO89SDO/axuiwhAhZfuvRtNYO4PZazdvqWW62v2Jojq9+eK1OW2JUYj1LCn87XFLGPe+vJS0liUcnDsFiNjPvylFGhyVEyDtjAlBKtQKeBHoD04GngPu01sf8HJthdh3eQIWj7u2frD5TDYgmcjmdTt5Y/yO/eX8dhaU2xvRqh93hkPo9QviIN1cAc4HPgZFACXAQ+C9wsR/jMlTOzvfqtHVISqVz6zQDoolM+46d5M4Fq/lk60/ER1t45tLh3DVaSecvhA95kwB6aq1fUkrdobWuAB5SSoV1OehKh61O23kDZhoQSWTKLylj0JxFFJdXcl5aB16cnkXPNnL7TQhf8yYBVCmlknG/EKmUSqO+r6PCRKW9bucPSO3/AHA6nZhMJtonxnFTZir927fkhpG9pYyDEH7iTQJ4GNcQkN2UUu8D2UDYjgn8/Z6v6rSZkGEC/anK7uCZb7aydl8Bb183FpPJxJzJw8+8ohCiWc6YALTWnyql1gGZgAW4TWtdt0BOmPjhwLI6bdmpkw2IJDJsOHCUW95eRe7+o7RLiGX/8VK6tmphdFhCRARv3gKqXaphiFIKrfWj9a4Qwly1f5x12vt0kLNRX7NV2Xn8i0389evNVDmczBzei6cnD6dNC7nVJkSgeHMLqPoNWCtwIbDaP+EYa8X2um//JETJyFG+5nA4GfePz1izt5BurVrwz2mZXNi3s9FhCRFxvLkF9Ej1aaXUn3G9Fhp2jpbWLSUwpt8MAyIJb2azieuG92Z41xT+ctFQEmOtRockRERqypfACUA3XwditIbe/mmf3COwgYSpL7cf5KklP7DwhnHER0dxx1nK6JCEiHjePAP4kZ9vjJuBlri+Bg4rK3fUHfkrwdIy8IGEmeNlFfz2w3W8smYXFrOJb3blM7Gf3O4RIhh4cwUwAzjs/rsTOK61LvawfEj6sWB9nbZsdbkBkYSP9zft5ZfvreFgcRlDOrVi7hXZDOsixduECBbeJIDXtNb9/B5JEJLSD003a1Euc5ZuISbKzOMXDeG+cQOwWqSMgxDBxJsEsEEpNRNYA5SdatRa7/VbVAFWVFZQp61lXAcDIgkfF/fvQs6eAl6cnkXf9slGhyOEqIc3CSDT/ac6J9DL9+EY47u8L+q0pbToYkAkoWvP0RPc92EucyZn0KN1AmN7t2fpXedLGQchgliDCUAp9Qut9b+11j0DGZARDh7bXadtYPdxgQ8kBDkcTl5YuZ3ff7KeE7Yq+rdP5tGJQwCk8xciyHm6ArgH+HegAjGSzVFSpy05LsWASEKLPlzErfNzWP7jYVrFRTPvylFcNzxsLgyFCHsyxiFgJYFKTpyeTrC0MjCa0PDexr1c+/oybFUOpg7qxnOXjaRDUpzRYQkhGsFTAhiglKp7b8RVGsKptQ6LU738orwanT+ASd5WOaOR3drQvVUCj100hMsHdTc6HCFEE3hKADuBiwIViFHW7PqoTlubBPlQqbbySjuPfbGRs3u3Z4LqRJeWLfjhgcmYzXKfX4hQ5SkBVGit9wQsEoOUlte9/z+05/kGRBK8Vvx4mFveXoU+UsyqvCNMUJ0ApPMXIsR5SgArAhaFgSrqDP9olQfAbiXllTz0yXc8v1IDcPeYvjzmfsNHCBH6GkwAWutfBjIQo9gpr9UStqNdNoo+XMSFL33F3mMn6dsuiZdmZHNWz3ZGhyWE8KGIfguovi+AhUuP1gm0jI3m2vE9eWj8IGKtFqNDEkL4WEQngNwfP63TFmuK3OEIF2zcQ+FJG7dm9yEmysKa31wk9XuECGMRnQAOHN1Rpy1bTQl8IAY7WFzK3e+tZeGmvbSKi+bqYT1JiLFK5y9EmIvYBFBUVkAVdQeB6Z7S34BojOF0Onl17S5++2Eux8sqGNOrHS/NyCYhRkboEiISRGwCyNkZ2QPAlFVWMWXeUr7cfpCEmCj+PnUkt2X3kVc7hYggEZsADhbtqtMWSQPAxFmjSI61ckHfTrwwLYturSL32YcQkSpiE0Dd1z2tYT8AzLb8IhZu2svvxw8E4NWrziLOapGqnUJEqIhMAPlFeXXa+rQbHvhAAqTS7mDOkh949PONVNgdTFCdGN61DfHREfnrF0K4RWQPsGrnwjptQ3qcY0Ak/rd+fyE3v72KDQeO0TEpjr9PHcnwrjIurxDCjwlAKWUGngcGAzbgZq31zmrzfwNc6Z78RGv9iL9iqa2o7GitFgvx0UmB2n3AzP50A3/5ahN2h5ObMlN5clIGLeOijQ5LCBEk/Pmi9xQgVmudDfwOePrUDKVUL+AaYBSQBZyvlBrkx1hOq7TbcFJZoy3RGp5j1lotJrq1bMHnt43npRnZ0vkLIWowOZ1Ov2xYKfU3YI3W+i339E9a687uv1uBZK11gXt6DXCt1np7fdvKzc3tAfzoi7gOlm2igG012izE0z/uYl9s3lAnKu3M10e5rn8KUWYTVQ4nlQ4ncVHyQZcQEa5nRkZGXu1Gfz4DSAKKqk3blVJRWusqrXUlUKCUMgFPAd811PlXl56eTkxMTKMDyc3NJSMjA4DXVnzgGtK+mrF9p4X8B2CLt/7EHe/msO94KQN69yQz7gSZI8L3wXZ9qv+eI4Ucc2Ro6jHbbDY2b97c4Hx/nhoWA4nV96W1rjo1oZSKBV53L3OnH+M4raisAIezolarNaQ7/8KTNn7xxgou+dfXHCwu4w8TBnJTVqrRYQkhQoA/rwBWAJOA+UqpLGDTqRnuM/8PgK+11n/1Yww1rNm1qE5bcgg//F30wz5umb+KIydsDO/ahrkzshnUScYzFkJ4x58JYCEwQSm1Etc4wjcope7FNdSkBTgbiFFKTXQv/3ut9So/xsPB43W//h2ZOsWfu/S7kvIqnrxkGPeM7UeUFG8TQjSC3xKA1toB3F6rufrT11h/7bshDux12kLp699Txdsm9u1Mh6Q4Jg3oyq6HLqNDUpzRoQkhQlDEnDJW2m3UefobQoe/u7CE81/4kpvfXsWsj9afbpfOXwjRVBHzJfDa3YvrtEUF/iKk0ewOB39frvnD4u8orbBzUb/OPH7REKPDEkKEgYhJANvzV9dpy+5zqQGReG/7kWKuf2MFq/cW0CY+hhenZ3PV0B5SvE0I4RMRkwDq3v6B3u0GGxCH90zAhgPHuHJoD/53ygjaJgT/FYsQInRETAKIIp4qSk9PJ8WkGBhNw9btK8QEZHRtQ1rbJDY/MImebRLPuJ4QQjRW6DwFbYayquM1On+AFnHB9b58aUUVDyzKJfvZxdzw1grsDtd4BdL5CyH8JSKuAA5Ufl+nrXfboYEPpAFLdx7i1vk57CosoXebRJ69bCQWc0TkZiGEgSIiAZRyvE5b95QBgQ+klpLySu5flMvcnB2YTSbuPbs/j1w4WAZqEUIERET2NNEkYLU0vqicr5lM8Lk+QHqHlsy9IpuR3YLzuYQQIjxFSAKo+QaQ0+SfEtjeOHKinE0Hj3FuWkcSYqx8dtt4urdqQXSUxbCYhBCRKSJuNFuo+bVsXHTgv551Op28uf5H0p/8kMtf/YaDxa6H0mltk6TzF0IYIiKuAGKIoZSS09PxcS0Duv/9x09y54LVfLzlJ+KsFh6bOIR28k6/EMJgEZEAqqiqMV1RcTIg+3U6nczN2cGsj9ZTXF7JuakdeGF6Fr1T5NVOIYTxIiIBWLFSfRiYmOiEgO373Q17AHhxehY3ZaZKGQchRNCIiASQQGdOcuT0dLfW/nsFtMruYOmufMb36YjJZOLlK0ZhNpvonBzvt30KIURTRMRD4OPsrjG9/VDdwnC+sOngMUY/9ykXvPgly3bnA9C1VQvp/IUQQSkirgDstfKcyWT16fZtVXae+HIzT3y1iSqHk2syetKvXbJP9yGEEL4WEQmgihM1pk+UFfps26v3HOGW+av44VARXZLjeX5aJhf37+Kz7QshhL9ERAIwEYOz2ptAcdEtfLbtBRv38sOhIm4f1YcnLh5KUmy0z7YthBD+FBEJwIypxmjAFnPzbgGt3nOEEV1TMJtNzL5gMJMHdGV0r3bNC1IIIQIsIh4Cx1LzIWzrFp2btJ3jZRXcOn8Vo/7vU15YuR2A+Ogo6fyFECEpIq4AKmt8BQBVjvJGb+ODzfu4a8FqDhaXMahjK0Z2l8JtQojQFvYJoNJuo6JWOeiyqtL6F67H4ZIy7nl/LfO/30O0xcyjFw7mgXPTsVoi4uJJCBHGwj4B7D78fZ22RGtLr9f/asch5n+/h6zuKcydkU3/Dt6vK4QQwSzsE8CO/HV12gZ2H+dxnX3HTpIYa6VlXDRXDu1BTJSFS9O7yChdQoiwEvY9msNurzGdHJ1Cclz99+8dDif/XKkZ+NQiHliUC4DJZGLqoG7S+Qshwk7YXwHYqspqTFc57fUut/1IMbfOX8Wy3YdpGRdNdo+2gQhPCCEME/YJwGIye5yusjt45putzP5sA+VVdqYM7Mrfp46kY5LU7xFChLewTwDmWrduak/rI8U8tPg7UlrE8NrUs7h8UPdAhieEEIYJ/wRQq/Cb2WTFVmWn4KSNzsnxDOjQkrdmjmVcantaxxs/ULwQQgRK2CeAk7aSGtNHy4rJ+NvHJMZEsfzuC7GYzUwd1M2g6IQQwjh+SwBKKTPwPDAYsAE3a613Vpt/C3AbUAU8prX+yB9x2Bw1K4EeOXGCrflF3HmWosLuIE7e7hFCRCh/9n5TgFitdTbwO+DpUzOUUh2AXwFnARcATyilfH7/Jb8oD3DWaLOY4Zu7LuC5qSOJs4b9BZAQQjTInwlgNPApgNY6Bxhebd5IYIXW2qa1LgJ2AoN8HcDaHz+u0zYmdbwUbxNCCPz7DCAJKKo2bVdKRWmtq+qZVwKccQitzZs3NyqAkrITddqcRxPJPZbbqO2EqtzcyDjO6uSYI4Mcs2/4MwEUA4nVps3uzr++eYlQq2JbPdLT04mJ8f5OUZeiNize9MLp6Qn9b6Jz6zSv1w9lubm5ZGRkGB1GQMkxRwY5Zu/ZbDaPJ87+vAW0ArgIQCmVBWyqNm8NMEYpFauUSgb6AY07vfdC++QeTBx4OwmmDkwceHvEdP5CCOENf14BLAQmKKVWAibgBqXUvcBOrfWHSqn/A5bhSkIPaa0bX6TfC+2Te9Azdgztk3v4Y/NCCBGy/JYAtNYO4PZazduqzZ8LzPXX/oUQQngmL8ELIUSEkgQghBARShKAEEJEKEkAQggRoUKlFoIFoKKioskbsNlsPgsmVMgxRwY55sjQlGOu1mda6ptvcjqd9bUHldzc3NG4XhkVQgjReGMyMjKW124MlSuAtcAY4CBQ/5iOQggharMAHXH1oXWExBWAEEII35OHwEIIEaEkAQghRISSBCCEEBFKEoAQQkQoSQBCCBGhQuU1UK8Ey0D0geTFMf8GuNI9+YnW+pHAR+k7Zzreast8DHygtX6h7lZCixe/44nAw7jKrucCd2mtQ/r1Pi+O+T7gasAB/EVrvdCQQP1AKZUJ/FVrPa5W+yTgT7j6r3nuisrNEm5XAFMweCB6A0yh4WPuBVwDjAKygPOVUj4feznAptDA8VbzGNAqkEH52RQa/h0nAk8Bl2itM4E8IMWAGH1tCg0fc0vgHiAbOB/438CH5x9KqQeAfwGxtdqtwDO4jvds4FalVPvm7i/cEoDhA9EbwNMx7wMu1Frb3WeEVsAvA+8EkKfjRSk1DddZ4aeBD81vPB3zKFyj7T2tlFoG5GutjwQ+RJ/zdMwngT1AC/cfR8Cj859dwNR62vvhGkzrmNa6AlgOjG3uzsItAdQ7EH0D87waiD4ENHjMWutKrXWBUsqklJoDfKe13m5IlL7T4PEqpdJx3Rb4kxGB+ZGnf9cpwDnALGAi8GulVJ8Ax+cPno4ZXCc3W4D1wP8FMjB/0lovACrrmeWX/ivcEoDPB6IPAZ6OGaVULPC6e5k7AxybP3g63uuAzsDXwPXAvUqpCwMbnl94OuZCYK3W+pDW+gTwLTAkwPH5g6djnoirvEFPoBswRSk1MsDxBZpf+q9wSwCGD0RvgAaPWSllAj4ANmitb9Nah0MdpQaPV2v9gNY60/3w7FXgb1rrcLgV5Onf9XogXSmV4j5DzsJ1ZhzqPB3zMaAMsLnHEj8OtAxwfIG2FUhTSrVWSkXjuv2zqrkbDau3gAiSgegDrMFjxlUI6mwgxv2mCMDvtdbN/odjII+/Y2ND85sz/bv+PfCZe9n5WutwOLE50zGPB3KUUg5c98O/MDBWv1FKXQ0kaK1fch//Z7j6r3la65+au30pBieEEBEq3G4BCSGE8JIkACGEiFCSAIQQIkJJAhBCiAglCUAIISJUuL0GKkKUUqoHsJ2677BP0lrva2Cd2QBa69nN2O/1wN+Ave6mOOAb4M7qH9R5ua1HgXXu1xSXaK3Pcbd/r7Ue0tQY3dtYCnQBTribkoDdwDVa63wP690KlGit32zO/kV4kgQggsmB5naUTfSh1vp6AKWUBVgK3AU825iNaK2rl6AYV619SHMDdLtZa70UTlfLfBe4F1cZiIaMwnU8QtQhCUAEPXeNn+eABKAd8LTW+v+qzbcC84B0d9PzWuu57mqJLwJdcRUM+73W+ktP+9Ja290fH/Vxb/sG4D7AiavU8i9xlSeub3+v4upsh7nXXa21zlRKnSrEtxcYqrXOV0q1xvUlenfgPOBR9zI/ArdorQvP8GNpgasO0Gr3vqa744xz/7kZiAYmA+cqpQ4C3zf25yHCmzwDEMGkk1Lq+2p/7ne334xr/IYRuAqfPV5rvVFAa631UGA8rpLf4DqDn6e1zsDVEb7oLp/cIKVUG1y1ZlYopQYCDwFna60H4qpC+bCH/QGgtf6V+7+Z1dqqgHeA6e6my4H3cZUw+B/gAvf2PgP+2kB4/1JKbXB35jm4vn59xn01cDuuktCD3du73925fwj8SWv9WVN+HiK8yRWACCYN3QK6D7jQXfJgEK4rgeo2A0op9RnwCT/fEhkP9HXfmwfXGXZvXGfC1U1WSn2Pq+SAGXgPeBPXbaBF1c7GXwJewdXB1re/M/kPrtr1fweuAv4AZOIqaLZEKQWu8h1HG1j/Zq31UqXUKGABrgF+KnAFcxkwSbk2Mg6or+6Ttz8PESEkAYhQMB9XAbBFwFv8PMIZAFrrQqXUAGACrgJi693TFuBcrfVRAKVUJ6C+B6annwFU5z6zrs4ERHnYn0da63XuYl4jgC5a65VKqUuB5Vrrye59xlKz6mN921nprmv1mlJqMK7BQ9biSjDfAhtx3aqqzdufh4gQcgtIhIIJuG5jfICruN2ph7W4/z4Z+C+uYSB/hetNma64ykLf6V6mP66OMb4R+12K6+qgtXv6Flxn6g3tr7ra9etPeR3Xffi33NOrgexqNfz/iGuErzP5G67nALfjel7hAP6C65gn4urswTV84Kk4mvvzEGFGEoAIBbOB5Uqp9biG88zDVQv+lMW4ygP/gKvs93ta603A3UCWUmoj8DYwU2td4u1OtdYbgSeAb5RS23Ddr/+Dh/1V9wGwwX1GX91/cdXr/697H4eAG4H5SqlNuB4g3+dFbDZczycexjWK1PfANlzloU/gergM8CXwoHuktGb9PET4kWqgQggRoeQKQAghIpQkACGEiFCSAIQQIkJJAhBCiAglCUAIISKUJAAhhIhQkgCEECJC/T9rqn+ea5XnHAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot\n",
    "\n",
    "# plot the roc curve for the model\n",
    "pyplot.plot([0,1], [0,1], linestyle='--', label='No Skill')\n",
    "pyplot.plot(fpr, tpr, marker='.', label='RF')\n",
    "# axis labels\n",
    "pyplot.xlabel('False Positive Rate')\n",
    "pyplot.ylabel('True Positive Rate')\n",
    "pyplot.legend()\n",
    "# show the plot\n",
    "pyplot.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}