titanic_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3 DEVELOP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn import tree\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import SVC\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "\n",
    "import sys\n",
    "sys.path.insert(0, './modules')\n",
    "from helpers import read_in_dataset, get_model_feature_importances, clf_performance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.2 load the preprocessed data from files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "***********************Reading in the X_train.csv dataset***********************\n",
      "\n",
      "it has 891 rows and 40 columns\n",
      "\n",
      "*************************it has the following columns\n",
      "**************************\n",
      "Index(['Age', 'Cabin_total', 'Parch', 'SibSp', 'norm_Fare',\n",
      "       'Cabin_firstletter_A', 'Cabin_firstletter_B', 'Cabin_firstletter_C',\n",
      "       'Cabin_firstletter_D', 'Cabin_firstletter_E', 'Cabin_firstletter_F',\n",
      "       'Cabin_firstletter_G', 'Cabin_firstletter_T', 'Cabin_firstletter_n',\n",
      "       'Embarked_C', 'Embarked_Q', 'Embarked_S', 'Pclass_1', 'Pclass_2',\n",
      "       'Pclass_3', 'Sex_female', 'Sex_male', 'Ticket_firstletter_A',\n",
      "       'Ticket_firstletter_C', 'Ticket_firstletter_F', 'Ticket_firstletter_L',\n",
      "       'Ticket_firstletter_None', 'Ticket_firstletter_P',\n",
      "       'Ticket_firstletter_S', 'Ticket_firstletter_W', 'name_title_adv_Col',\n",
      "       'name_title_adv_Dr', 'name_title_adv_Major', 'name_title_adv_Master',\n",
      "       'name_title_adv_Miss', 'name_title_adv_Mr', 'name_title_adv_Mrs',\n",
      "       'name_title_adv_Rev', 'name_title_adv_title_nonsurvive',\n",
      "       'name_title_adv_title_survive'],\n",
      "      dtype='object')\n",
      "\n",
      "************************the first 5 rows looks like this************************\n",
      "\n",
      "        Age  Cabin_total     Parch     SibSp  norm_Fare  Cabin_firstletter_A  \\\n",
      "0 -0.584387    -0.488483 -0.473674  0.432793  -0.879741                    0   \n",
      "1  0.621365     1.340249 -0.473674  0.432793   1.361220                    0   \n",
      "2 -0.282949    -0.488483 -0.473674 -0.474545  -0.798540                    0   \n",
      "3  0.395286     1.340249 -0.473674  0.432793   1.062038                    0   \n",
      "4  0.395286    -0.488483 -0.473674 -0.474545  -0.784179                    0   \n",
      "\n",
      "   Cabin_firstletter_B  Cabin_firstletter_C  Cabin_firstletter_D  \\\n",
      "0                    0                    0                    0   \n",
      "1                    0                    1                    0   \n",
      "2                    0                    0                    0   \n",
      "3                    0                    1                    0   \n",
      "4                    0                    0                    0   \n",
      "\n",
      "   Cabin_firstletter_E  ...  name_title_adv_Col  name_title_adv_Dr  \\\n",
      "0                    0  ...                   0                  0   \n",
      "1                    0  ...                   0                  0   \n",
      "2                    0  ...                   0                  0   \n",
      "3                    0  ...                   0                  0   \n",
      "4                    0  ...                   0                  0   \n",
      "\n",
      "   name_title_adv_Major  name_title_adv_Master  name_title_adv_Miss  \\\n",
      "0                     0                      0                    0   \n",
      "1                     0                      0                    0   \n",
      "2                     0                      0                    1   \n",
      "3                     0                      0                    0   \n",
      "4                     0                      0                    0   \n",
      "\n",
      "   name_title_adv_Mr  name_title_adv_Mrs  name_title_adv_Rev  \\\n",
      "0                  1                   0                   0   \n",
      "1                  0                   1                   0   \n",
      "2                  0                   0                   0   \n",
      "3                  0                   1                   0   \n",
      "4                  1                   0                   0   \n",
      "\n",
      "   name_title_adv_title_nonsurvive  name_title_adv_title_survive  \n",
      "0                                0                             0  \n",
      "1                                0                             0  \n",
      "2                                0                             0  \n",
      "3                                0                             0  \n",
      "4                                0                             0  \n",
      "\n",
      "[5 rows x 40 columns]\n",
      "\n",
      "***********************Reading in the X_test.csv dataset************************\n",
      "\n",
      "it has 418 rows and 40 columns\n",
      "\n",
      "*************************it has the following columns\n",
      "**************************\n",
      "Index(['Age', 'Cabin_total', 'Parch', 'SibSp', 'norm_Fare',\n",
      "       'Cabin_firstletter_A', 'Cabin_firstletter_B', 'Cabin_firstletter_C',\n",
      "       'Cabin_firstletter_D', 'Cabin_firstletter_E', 'Cabin_firstletter_F',\n",
      "       'Cabin_firstletter_G', 'Cabin_firstletter_T', 'Cabin_firstletter_n',\n",
      "       'Embarked_C', 'Embarked_Q', 'Embarked_S', 'Pclass_1', 'Pclass_2',\n",
      "       'Pclass_3', 'Sex_female', 'Sex_male', 'Ticket_firstletter_A',\n",
      "       'Ticket_firstletter_C', 'Ticket_firstletter_F', 'Ticket_firstletter_L',\n",
      "       'Ticket_firstletter_None', 'Ticket_firstletter_P',\n",
      "       'Ticket_firstletter_S', 'Ticket_firstletter_W', 'name_title_adv_Col',\n",
      "       'name_title_adv_Dr', 'name_title_adv_Major', 'name_title_adv_Master',\n",
      "       'name_title_adv_Miss', 'name_title_adv_Mr', 'name_title_adv_Mrs',\n",
      "       'name_title_adv_Rev', 'name_title_adv_title_nonsurvive',\n",
      "       'name_title_adv_title_survive'],\n",
      "      dtype='object')\n",
      "\n",
      "************************the first 5 rows looks like this************************\n",
      "\n",
      "        Age  Cabin_total     Parch     SibSp  norm_Fare  Cabin_firstletter_A  \\\n",
      "0  0.357607    -0.488483 -0.473674 -0.474545  -0.809683                    0   \n",
      "1  1.299600    -0.488483 -0.473674  0.432793  -0.911513                    0   \n",
      "2  2.429993    -0.488483 -0.473674 -0.474545  -0.612461                    0   \n",
      "3 -0.207590    -0.488483 -0.473674 -0.474545  -0.716562                    0   \n",
      "4 -0.584387    -0.488483  0.767630  0.432793  -0.387631                    0   \n",
      "\n",
      "   Cabin_firstletter_B  Cabin_firstletter_C  Cabin_firstletter_D  \\\n",
      "0                    0                    0                    0   \n",
      "1                    0                    0                    0   \n",
      "2                    0                    0                    0   \n",
      "3                    0                    0                    0   \n",
      "4                    0                    0                    0   \n",
      "\n",
      "   Cabin_firstletter_E  ...  name_title_adv_Col  name_title_adv_Dr  \\\n",
      "0                    0  ...                   0                  0   \n",
      "1                    0  ...                   0                  0   \n",
      "2                    0  ...                   0                  0   \n",
      "3                    0  ...                   0                  0   \n",
      "4                    0  ...                   0                  0   \n",
      "\n",
      "   name_title_adv_Major  name_title_adv_Master  name_title_adv_Miss  \\\n",
      "0                     0                      0                    0   \n",
      "1                     0                      0                    0   \n",
      "2                     0                      0                    0   \n",
      "3                     0                      0                    0   \n",
      "4                     0                      0                    0   \n",
      "\n",
      "   name_title_adv_Mr  name_title_adv_Mrs  name_title_adv_Rev  \\\n",
      "0                  1                   0                   0   \n",
      "1                  0                   1                   0   \n",
      "2                  1                   0                   0   \n",
      "3                  1                   0                   0   \n",
      "4                  0                   1                   0   \n",
      "\n",
      "   name_title_adv_title_nonsurvive  name_title_adv_title_survive  \n",
      "0                                0                             0  \n",
      "1                                0                             0  \n",
      "2                                0                             0  \n",
      "3                                0                             0  \n",
      "4                                0                             0  \n",
      "\n",
      "[5 rows x 40 columns]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(891, 1)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train = read_in_dataset('X_train.csv', verbose = True)\n",
    "X_test = read_in_dataset('X_test.csv', verbose = True)\n",
    "test = read_in_dataset('data/test.csv')\n",
    "Y_train = pd.read_csv('Y_train.csv',header=None)\n",
    "Y_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.3 Try try different models using the default parameters\n",
    "* naive bayes 0.4668\n",
    "* Logistic regression 0.81933, turned to 0.8215\n",
    "* tree 0.7924\n",
    "* knn 0.8149, turned to 0.8249\n",
    "* RandomForest, 0.81, turned to 0.8339, to 0.8372\n",
    "* SVC, 0.8306, to 0.8350\n",
    "* xgboost, 0.8305, turned to 0.8451"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.53072626 0.46368715 0.45505618 0.41573034 0.46892655]\n",
      "0.4668252956694957\n"
     ]
    }
   ],
   "source": [
    "#Use Naive Bayes as a baseline\n",
    "gnb = GaussianNB()\n",
    "cv = cross_val_score(gnb, X_train, Y_train.values.ravel(), cv = 5)\n",
    "print(cv)\n",
    "print(cv.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.82122905 0.82681564 0.78089888 0.81460674 0.85310734]\n",
      "0.8193315310695454\n"
     ]
    }
   ],
   "source": [
    "lr = LogisticRegression(max_iter = 2000, solver='lbfgs')\n",
    "cv = cross_val_score(lr, X_train, Y_train.values.ravel(), cv = 5)\n",
    "print(cv)\n",
    "print(cv.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.7877095  0.78212291 0.83707865 0.76404494 0.79096045]\n",
      "0.7923832899435312\n"
     ]
    }
   ],
   "source": [
    "dt = tree.DecisionTreeClassifier(random_state = 1)\n",
    "cv = cross_val_score(dt, X_train, Y_train.values.ravel(), cv = 5)\n",
    "print(cv)\n",
    "print(cv.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.79888268 0.77094972 0.83707865 0.83146067 0.83615819]\n",
      "0.8149059840335457\n"
     ]
    }
   ],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "cv = cross_val_score(knn, X_train, Y_train.values.ravel(), cv = 5)\n",
    "print(cv)\n",
    "print(cv.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.79329609 0.80446927 0.84269663 0.78089888 0.83615819]\n",
      "0.8115038121673729\n"
     ]
    }
   ],
   "source": [
    "rf = RandomForestClassifier(random_state = 1, n_estimators = 10)\n",
    "cv = cross_val_score(rf, X_train, Y_train.values.ravel(), cv = 5)\n",
    "print(cv)\n",
    "print(cv.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.84357542 0.82681564 0.8258427  0.79213483 0.86440678]\n",
      "0.8305550738406838\n"
     ]
    }
   ],
   "source": [
    "svc = SVC(probability = True, gamma = 'auto')\n",
    "cv = cross_val_score(svc, X_train, Y_train.values.ravel(), cv = 5)\n",
    "print(cv)\n",
    "print(cv.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.82681564 0.82122905 0.86516854 0.82022472 0.81920904]\n",
      "0.8305293981424839\n"
     ]
    }
   ],
   "source": [
    "xgb = XGBClassifier(random_state = 1)\n",
    "cv = cross_val_score(xgb, X_train, Y_train.values.ravel(), cv = 5)\n",
    "print(cv)\n",
    "print(cv.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "voting_clf = VotingClassifier(estimators = [('gnb', gnb), ('lr', lr), ('dt', dt), ('knn', knn), ('rf', rf),  \n",
    "                                            ('svc', svc), ('xgb', xgb)], voting = 'soft')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.82681564 0.81564246 0.84269663 0.80898876 0.83615819]\n",
      "0.8260603371814963\n"
     ]
    }
   ],
   "source": [
    "cv = cross_val_score(voting_clf, X_train, Y_train.values.ravel(), cv = 5)\n",
    "print(cv)\n",
    "print(cv.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "voting_clf.fit(X_train, Y_train.values.ravel())\n",
    "Y_hat_base_vc = voting_clf.predict(X_test).astype(int)\n",
    "basic_submission = {'PassengerId': test.PassengerId, 'Survived': Y_hat_base_vc}\n",
    "base_submission = pd.DataFrame(data = basic_submission)\n",
    "base_submission.to_csv('base_submission.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dtype('int64')"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_hat_base_vc.dtype"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.4 Model Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 40 candidates, totalling 200 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression\n",
      "Best Score: 0.8215488215488216\n",
      "Best Parameters: {'C': 78.47599703514607, 'max_iter': 2000, 'penalty': 'l1', 'solver': 'liblinear'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 200 out of 200 | elapsed:    1.9s finished\n"
     ]
    }
   ],
   "source": [
    "lr = LogisticRegression()\n",
    "param_grid = {'max_iter': [2000],\n",
    "             'penalty': ['l1', 'l2'],\n",
    "             'C': np.logspace(-4, 4, 20),\n",
    "             'solver': ['liblinear']}\n",
    "\n",
    "clf_lr = GridSearchCV(lr, param_grid = param_grid, cv = 5, verbose = True, n_jobs = -1)\n",
    "best_clf_lr = clf_lr.fit(X_train, Y_train.values.ravel())\n",
    "clf_performance(best_clf_lr, 'Logistic Regression')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 48 candidates, totalling 240 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KNN\n",
      "Best Score: 0.8249158249158249\n",
      "Best Parameters: {'algorithm': 'auto', 'n_neighbors': 7, 'p': 2, 'weights': 'uniform'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 240 out of 240 | elapsed:    1.9s finished\n"
     ]
    }
   ],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "param_grid = {'n_neighbors': [3, 5, 7, 9],\n",
    "             'weights': ['uniform', 'distance'],\n",
    "             'algorithm': ['auto', 'ball_tree', 'kd_tree'],\n",
    "             'p': [1,2]}\n",
    "clf_knn = GridSearchCV(knn, param_grid = param_grid, cv = 5, verbose = True, n_jobs = -1)\n",
    "best_clf_knn = clf_knn.fit(X_train, Y_train.values.ravel())\n",
    "clf_performance(best_clf_knn, 'KNN')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 55 candidates, totalling 275 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:    2.7s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  1.6min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVC\n",
      "Best Score: 0.835016835016835\n",
      "Best Parameters: {'C': 10, 'degree': 2, 'kernel': 'poly'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 275 out of 275 | elapsed:  2.8min finished\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py:814: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "svc = SVC(probability = True)\n",
    "param_grid = turned_parameters = [{'kernel': ['rbf'], 'gamma': [.1, .5, 1, 2, 5, 10],\n",
    "                                  'C': [.1, 1, 10, 100, 1000]},\n",
    "                                 {'kernel': ['linear'], 'C': [.1, 1, 10, 100, 1000]}, \n",
    "                                 {'kernel': ['poly'], 'degree': [2, 3, 4, 5], 'C': [.1, 1, 10, 100, 1000]}]\n",
    "clf_svc = GridSearchCV(svc, param_grid = param_grid, cv = 5, verbose = True, n_jobs = -1)\n",
    "best_clf_svc = clf_svc.fit(X_train, Y_train.values.ravel())\n",
    "clf_performance(best_clf_svc, 'SVC')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 100 candidates, totalling 500 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   19.5s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  1.9min\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  4.0min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  4.6min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest\n",
      "Best Score: 0.835016835016835\n",
      "Best Parameters: {'n_estimators': 1000, 'min_samples_split': 10, 'min_samples_leaf': 1, 'max_features': 'auto', 'max_depth': 75, 'bootstrap': True}\n"
     ]
    }
   ],
   "source": [
    "#use random search to narrow down the space for random forest\n",
    "rf = RandomForestClassifier(random_state = 1)\n",
    "param_grid = {'n_estimators': [100, 500, 1000],\n",
    "             'bootstrap': [True, False],\n",
    "             'max_depth': [3, 5, 10, 20, 50, 75, 100, None],\n",
    "             'max_features': ['auto', 'sqrt'],\n",
    "             'min_samples_leaf': [1, 2, 4, 10],\n",
    "             'min_samples_split': [2, 5, 10]}\n",
    "clf_rf = RandomizedSearchCV(rf, param_distributions = param_grid, n_iter = 100, cv = 5, verbose = True, n_jobs = -1)\n",
    "best_clf_rf = clf_rf.fit(X_train, Y_train.values.ravel())\n",
    "clf_performance(best_clf_rf, 'Random Forest')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 100 candidates, totalling 500 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   40.2s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  2.6min\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  5.7min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  6.4min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest 2\n",
      "Best Score: 0.8372615039281706\n",
      "Best Parameters: {'n_estimators': 900, 'min_samples_split': 11, 'min_samples_leaf': 2, 'max_features': 'auto', 'max_depth': 25, 'criterion': 'entropy', 'bootstrap': False}\n"
     ]
    }
   ],
   "source": [
    "rf = RandomForestClassifier(random_state = 1)\n",
    "param_grid = {'n_estimators': [800, 900, 1000, 1100, 1200],\n",
    "             'criterion': ['gini','entropy'],\n",
    "             'bootstrap': [False],\n",
    "             'max_depth': [15, 20, 25],\n",
    "             'max_features': ['auto', 'sqrt', 10],\n",
    "             'min_samples_leaf': [2, 3],\n",
    "             'min_samples_split': [9, 10, 11]}\n",
    "clf_rf = RandomizedSearchCV(rf, param_distributions = param_grid, n_iter = 100, cv = 5, verbose = True, n_jobs = -1)\n",
    "best_clf_rf = clf_rf.fit(X_train, Y_train.values.ravel())\n",
    "clf_performance(best_clf_rf, 'Random Forest 2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7fa8f62b1ed0>"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_rf = best_clf_rf.best_estimator_.fit(X_train, Y_train.values.ravel())\n",
    "feat_importances = pd.Series(best_rf.feature_importances_, index = X_train.columns)\n",
    "feat_importances.nlargest(20).plot(kind = 'barh')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "name_title_adv_Mr          0.134457\n",
      "norm_Fare                  0.130868\n",
      "Sex_male                   0.113910\n",
      "Age                        0.110472\n",
      "Sex_female                 0.106597\n",
      "Pclass_3                   0.059647\n",
      "SibSp                      0.041508\n",
      "name_title_adv_Mrs         0.033677\n",
      "Cabin_total                0.032582\n",
      "Cabin_firstletter_n        0.029096\n",
      "name_title_adv_Miss        0.028958\n",
      "Pclass_1                   0.025715\n",
      "Pclass_2                   0.021339\n",
      "Parch                      0.020630\n",
      "name_title_adv_Master      0.015554\n",
      "Embarked_S                 0.011729\n",
      "Ticket_firstletter_None    0.011040\n",
      "Embarked_C                 0.010168\n",
      "Ticket_firstletter_S       0.007420\n",
      "Embarked_Q                 0.006616\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print(feat_importances.nlargest(20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 100 candidates, totalling 500 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   14.7s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:   40.6s\n",
      "[Parallel(n_jobs=-1)]: Done 457 tasks      | elapsed:  1.7min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XGB random\n",
      "Best Score: 0.8417508417508418\n",
      "Best Parameters: {'subsample': 0.7, 'sampling_method': 'uniform', 'reg_lambda': 2, 'reg_alpha': 0.5, 'n_estimators': 100, 'min_child_weight': 0.01, 'max_depth': 10, 'learning_rate': 0.3, 'gamma': 0.01, 'colsample_bytree': 0.2}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  1.9min finished\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py:814: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "xgb = XGBClassifier(random_state = 1)\n",
    "param_grid = {\n",
    "    'n_estimators': [20, 50, 100, 250, 500, 1000],\n",
    "    'colsample_bytree': [0.2, 0.5, 0.7, 0.8, 1],\n",
    "    'max_depth': [2, 5, 10, 15, 20, 25, None],\n",
    "    'reg_alpha': [0, 0.5, 1],\n",
    "    'reg_lambda': [1, 1.5, 2],\n",
    "    'subsample': [0.5, 0.6, 0.7, 0.8, 0.9],\n",
    "    'learning_rate': [.01, 0.1, 0.2, 0.3, 0.5, 0.7, 0.9],\n",
    "    'gamma': [0, .01, .1, 1, 10, 100],\n",
    "    'min_child_weight': [0, .01, .1, 1, 10, 100],\n",
    "    'sampling_method': ['uniform']\n",
    "}\n",
    "clf_xgb_rnd = RandomizedSearchCV(xgb, param_distributions = param_grid, n_iter = 100, cv = 5, \n",
    "                                 verbose = True, n_jobs = -1)\n",
    "best_clf_xgb_rnd = clf_xgb_rnd.fit(X_train, Y_train.values.ravel())\n",
    "clf_performance(best_clf_xgb_rnd, 'XGB random')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_hat_xgb_rnd = best_clf_xgb_rnd.best_estimator_.predict(X_test).astype(int)\n",
    "xgb_rnd_submission = {'PassengerId': test.PassengerId, 'Survived': Y_hat_xgb_rnd}\n",
    "submission_xgb_rnd = pd.DataFrame(data = xgb_rnd_submission)\n",
    "submission_xgb_rnd.to_csv('xgb_rnd_submission.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "voting_clf_hard:  [0.84357542 0.81564246 0.84269663 0.79775281 0.85310734]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "voting_clf_hard mean:  0.8305549319859976\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "voting_clf_soft:  [0.83798883 0.81564246 0.8258427  0.79775281 0.8700565 ]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "voting_clf_soft mean:  0.830580253047482\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "voting_clf_all:  [0.83240223 0.81564246 0.84269663 0.79775281 0.8700565 ]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "voting_clf_all mean:  0.8317101256229638\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "voting_clf_xgb:  [0.83798883 0.82681564 0.84269663 0.79775281 0.87570621]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "voting_clf_xgb mean:  0.8361920244330513\n"
     ]
    }
   ],
   "source": [
    "best_lr = best_clf_lr.best_estimator_\n",
    "best_knn = best_clf_knn.best_estimator_\n",
    "best_svc = best_clf_svc.best_estimator_\n",
    "best_rf = best_clf_rf.best_estimator_\n",
    "best_xgb = best_clf_xgb_rnd.best_estimator_\n",
    "\n",
    "voting_clf_hard = VotingClassifier(estimators = [('knn', best_knn), ('rf', best_rf), ('svc', best_svc)], \n",
    "                                                 voting = 'hard')\n",
    "voting_clf_soft = VotingClassifier(estimators = [('knn', best_knn), ('rf', best_rf), ('svc', best_svc)], \n",
    "                                                 voting = 'soft')\n",
    "voting_clf_all = VotingClassifier(estimators = [('knn', best_knn), ('rf', best_rf), ('svc', best_svc), \n",
    "                                                ('lr', best_lr)], voting = 'soft')\n",
    "voting_clf_xgb = VotingClassifier(estimators = [('knn', best_knn), ('rf', best_rf), ('svc', best_svc), \n",
    "                                                ('xgb', best_xgb),('lr', best_lr)], voting = 'soft')\n",
    "print('voting_clf_hard: ', cross_val_score(voting_clf_hard, X_train, Y_train.values.ravel(), cv = 5))\n",
    "print('voting_clf_hard mean: ', cross_val_score(voting_clf_hard, X_train, Y_train.values.ravel(), cv = 5).mean())\n",
    "\n",
    "print('voting_clf_soft: ', cross_val_score(voting_clf_soft, X_train, Y_train.values.ravel(), cv = 5))\n",
    "print('voting_clf_soft mean: ', cross_val_score(voting_clf_soft, X_train, Y_train.values.ravel(), cv = 5).mean())\n",
    "\n",
    "print('voting_clf_all: ', cross_val_score(voting_clf_all, X_train, Y_train.values.ravel(), cv = 5))\n",
    "print('voting_clf_all mean: ', cross_val_score(voting_clf_all, X_train, Y_train.values.ravel(), cv = 5).mean())\n",
    "\n",
    "print('voting_clf_xgb: ', cross_val_score(voting_clf_xgb, X_train, Y_train.values.ravel(), cv = 5))\n",
    "print('voting_clf_xgb mean: ', cross_val_score(voting_clf_xgb, X_train, Y_train.values.ravel(), cv = 5).mean())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. DEPLOY using the best model xgboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#clf_xgb_rnd mean has the best estimate\n",
    "Y_hat_xgb_rnd = best_clf_xgb_rnd.best_estimator_.predict(X_test).astype(int)\n",
    "xgb_rnd_submission = {'PassengerId': test.PassengerId, 'Survived': Y_hat_xgb_rnd}\n",
    "submission_xgb_rnd = pd.DataFrame(data = xgb_rnd_submission)\n",
    "submission_xgb_rnd.to_csv('xgb_rnd_submission.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_clf_xgb_rnd = best_clf_xgb_rnd.best_estimator_.fit(X_train, Y_train.values.ravel())\n",
    "feat_importances = pd.Series(best_clf_xgb_rnd.feature_importances_, index = X_train.columns)\n",
    "feat_importances.nlargest(20).plot(kind = 'barh')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sex_male                 0.375361\n",
      "name_title_adv_Master    0.078903\n",
      "Pclass_3                 0.068306\n",
      "Pclass_1                 0.029994\n",
      "Sex_female               0.029295\n",
      "norm_Fare                0.027792\n",
      "Embarked_S               0.027290\n",
      "SibSp                    0.020011\n",
      "name_title_adv_Mr        0.019461\n",
      "Age                      0.019317\n",
      "Cabin_firstletter_G      0.018700\n",
      "Pclass_2                 0.017393\n",
      "name_title_adv_Miss      0.017327\n",
      "Cabin_firstletter_C      0.017247\n",
      "Embarked_C               0.016396\n",
      "Cabin_firstletter_E      0.015445\n",
      "Parch                    0.014998\n",
      "Cabin_firstletter_n      0.014038\n",
      "Ticket_firstletter_S     0.011664\n",
      "Embarked_Q               0.011504\n",
      "dtype: float32\n"
     ]
    }
   ],
   "source": [
    "print(feat_importances.nlargest(20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3AAAAHOCAYAAADZtev7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAABLPUlEQVR4nO3deZhlVXn3/e+PRgUVFAUnZhEHRBBFxCHGAQSjAeMQcDYSiXEiIYNgFBIcMTGJUXwUAwaNiCgaW0VBFHFAkAYRBOUREAfe+IgDQtQwyP3+sfehTxfVXdVQ3Xuv6u/nuurqs/c+p+ruVXXWWffaa0hVIUmSJEkav/WGDkCSJEmSND8mcJIkSZLUCBM4SZIkSWqECZwkSZIkNcIETpIkSZIasf7QAcy06aab1jbbbDN0GJIkSZI0iHPPPfdnVbXZbNdGl8Bts802LFu2bOgwJEmSJGkQSX6wsmsOoZQkSZKkRpjASZIkSVIjTOAkSZIkqREmcJIkSZLUCBM4SZIkSWqECZwkSZIkNcIETpIkSZIaYQInSZIkSY0wgZMkSZKkRpjASZIkSVIjTOAkSZIkqREmcJIkSZLUCBM4SZIkSWqECZwkSZIkNWL9oQO4LbY55DML/j2veOtTF/x7SpIkSdJC8A6cJEmSJDXCBE6SJEmSGmECJ0mSJEmNMIGTJEmSpEaYwEmSJElSI0zgJEmSJKkRJnCSJEmS1AgTOEmSJElqhAmcJEmSJDXCBE6SJEmSGmECJ0mSJEmNMIGTJEmSpEaYwEmSJElSI0zgJEmSJKkRJnCSJEmS1AgTOEmSJElqhAmcJEmSJDViXglckr2TXJLk0iSHzHL9ZUkuTHJ+kq8m2aE/v02S3/bnz0/ynoX+D0iSJEnSumL9uZ6QZAlwFLAn8GPgnCRLq+riqacdX1Xv6Z+/D/DPwN79tcuq6qELGrUkSZIkrYPmcwduN+DSqrq8qq4HTgD2nX5CVV0zdXgnoBYuREmSJEkSzC+B2xz40dTxj/tzK0jyiiSXAW8DXj11adsk30xyRpLfm+0HJDkwybIky6666qrVCF+SJEmS1h0LtohJVR1VVdsBrwFe15/+b2CrqtoFOBg4PsnGs7z26Kratap23WyzzRYqJEmSJElaVOaTwF0JbDl1vEV/bmVOAJ4OUFXXVdXP+8fnApcB979VkUqSJEnSOm4+Cdw5wPZJtk1ye2B/YOn0E5JsP3X4VOB7/fnN+kVQSHJfYHvg8oUIXJIkSZLWNXOuQllVNyZ5JXAKsAQ4tqouSnIEsKyqlgKvTLIHcAPwS+BF/csfBxyR5AbgJuBlVfWLNfEfkSRJkqTFbs4EDqCqTgZOnnHusKnHB63kdScBJ92WACVJkiRJnQVbxESSJEmStGaZwEmSJElSI0zgJEmSJKkRJnCSJEmS1AgTOEmSJElqhAmcJEmSJDXCBE6SJEmSGmECJ0mSJEmNMIGTJEmSpEaYwEmSJElSI0zgJEmSJKkRJnCSJEmS1AgTOEmSJElqhAmcJEmSJDXCBE6SJEmSGmECJ0mSJEmNMIGTJEmSpEaYwEmSJElSI0zgJEmSJKkRJnCSJEmS1AgTOEmSJElqhAmcJEmSJDXCBE6SJEmSGmECJ0mSJEmNMIGTJEmSpEaYwEmSJElSI0zgJEmSJKkRJnCSJEmS1AgTOEmSJElqhAmcJEmSJDXCBE6SJEmSGmECJ0mSJEmNmFcCl2TvJJckuTTJIbNcf1mSC5Ocn+SrSXaYunZo/7pLkuy1kMFLkiRJ0rpkzgQuyRLgKOApwA7Ac6YTtN7xVfWQqnoo8Dbgn/vX7gDsDzwY2Bt4d//9JEmSJEmraT534HYDLq2qy6vqeuAEYN/pJ1TVNVOHdwKqf7wvcEJVXVdV3wcu7b+fJEmSJGk1rT+P52wO/Gjq+MfAI2c+KckrgIOB2wNPnHrtWTNeu/ksrz0QOBBgq622mk/ckiRJkrTOWbBFTKrqqKraDngN8LrVfO3RVbVrVe262WabLVRIkiRJkrSozCeBuxLYcup4i/7cypwAPP1WvlaSJEmStBLzSeDOAbZPsm2S29MtSrJ0+glJtp86fCrwvf7xUmD/JHdIsi2wPfCN2x62JEmSJK175pwDV1U3JnklcAqwBDi2qi5KcgSwrKqWAq9MsgdwA/BL4EX9ay9KciJwMXAj8Iqq+t0a+r9IkiRJ0qI2n0VMqKqTgZNnnDts6vFBq3jtm4A33doAJUmSJEmdBVvERJIkSZK0ZpnASZIkSVIjTOAkSZIkqREmcJIkSZLUCBM4SZIkSWqECZwkSZIkNcIETpIkSZIaYQInSZIkSY0wgZMkSZKkRpjASZIkSVIjTOAkSZIkqREmcJIkSZLUCBM4SZIkSWqECZwkSZIkNcIETpIkSZIaYQInSZIkSY0wgZMkSZKkRpjASZIkSVIjTOAkSZIkqREmcJIkSZLUCBM4SZIkSWqECZwkSZIkNcIETpIkSZIaYQInSZIkSY0wgZMkSZKkRpjASZIkSVIjTOAkSZIkqREmcJIkSZLUCBM4SZIkSWqECZwkSZIkNcIETpIkSZIaYQInSZIkSY2YVwKXZO8klyS5NMkhs1w/OMnFSS5I8oUkW09d+12S8/uvpQsZvCRJkiStS9af6wlJlgBHAXsCPwbOSbK0qi6eeto3gV2r6jdJ/hx4G7Bff+23VfXQhQ1bkiRJktY987kDtxtwaVVdXlXXAycA+04/oapOr6rf9IdnAVssbJiSJEmSpPkkcJsDP5o6/nF/bmUOAD47dbxBkmVJzkry9NUPUZIkSZIE8xhCuTqSPB/YFfj9qdNbV9WVSe4LfDHJhVV12YzXHQgcCLDVVlstZEiSJEmStGjM5w7clcCWU8db9OdWkGQP4O+Afarqusn5qrqy//dy4EvALjNfW1VHV9WuVbXrZptttlr/AUmSJElaV8wngTsH2D7JtkluD+wPrLCaZJJdgPfSJW8/nTq/SZI79I83BR4DTC9+IkmSJEmapzmHUFbVjUleCZwCLAGOraqLkhwBLKuqpcA/AncGPpoE4IdVtQ/wIOC9SW6iSxbfOmP1SkmSJEnSPM1rDlxVnQycPOPcYVOP91jJ684EHnJbApQkSZIkdea1kbckSZIkaXgmcJIkSZLUCBM4SZIkSWqECZwkSZIkNcIETpIkSZIaYQInSZIkSY0wgZMkSZKkRpjASZIkSVIjTOAkSZIkqREmcJIkSZLUCBM4SZIkSWqECZwkSZIkNcIETpIkSZIaYQInSZIkSY0wgZMkSZKkRpjASZIkSVIjTOAkSZIkqREmcJIkSZLUCBM4SZIkSWqECZwkSZIkNcIETpIkSZIaYQInSZIkSY0wgZMkSZKkRpjASZIkSVIjTOAkSZIkqREmcJIkSZLUCBM4SZIkSWqECZwkSZIkNcIETpIkSZIaYQInSZIkSY0wgZMkSZKkRpjASZIkSVIjTOAkSZIkqRHzSuCS7J3kkiSXJjlklusHJ7k4yQVJvpBk66lrL0ryvf7rRQsZvCRJkiStS+ZM4JIsAY4CngLsADwnyQ4znvZNYNeq2gn4GPC2/rV3Aw4HHgnsBhyeZJOFC1+SJEmS1h3zuQO3G3BpVV1eVdcDJwD7Tj+hqk6vqt/0h2cBW/SP9wI+X1W/qKpfAp8H9l6Y0CVJkiRp3TKfBG5z4EdTxz/uz63MAcBnV+e1SQ5MsizJsquuumoeIUmSJEnSumdBFzFJ8nxgV+AfV+d1VXV0Ve1aVbtuttlmCxmSJEmSJC0a80ngrgS2nDreoj+3giR7AH8H7FNV163OayVJkiRJc5tPAncOsH2SbZPcHtgfWDr9hCS7AO+lS95+OnXpFODJSTbpFy95cn9OkiRJkrSa1p/rCVV1Y5JX0iVeS4Bjq+qiJEcAy6pqKd2QyTsDH00C8MOq2qeqfpHkDXRJIMARVfWLNfI/kSRJkqRFbs4EDqCqTgZOnnHusKnHe6zitccCx97aACVJkiRJnQVdxESSJEmStOaYwEmSJElSI0zgJEmSJKkRJnCSJEmS1AgTOEmSJElqhAmcJEmSJDXCBE6SJEmSGmECJ0mSJEmNMIGTJEmSpEaYwEmSJElSI0zgJEmSJKkRJnCSJEmS1AgTOEmSJElqhAmcJEmSJDXCBE6SJEmSGmECJ0mSJEmNMIGTJEmSpEaYwEmSJElSI0zgJEmSJKkRJnCSJEmS1AgTOEmSJElqhAmcJEmSJDXCBE6SJEmSGmECJ0mSJEmNMIGTJEmSpEaYwEmSJElSI0zgJEmSJKkRJnCSJEmS1AgTOEmSJElqhAmcJEmSJDXCBE6SJEmSGmECJ0mSJEmNmFcCl2TvJJckuTTJIbNcf1yS85LcmORZM679Lsn5/dfShQpckiRJktY168/1hCRLgKOAPYEfA+ckWVpVF0897YfAi4G/nuVb/LaqHnrbQ5UkSZKkdducCRywG3BpVV0OkOQEYF/g5gSuqq7or920BmKUJEmSJDG/IZSbAz+aOv5xf26+NkiyLMlZSZ4+2xOSHNg/Z9lVV121Gt9akiRJktYda2MRk62ralfgucC/Jtlu5hOq6uiq2rWqdt1ss83WQkiSJEmS1J75JHBXAltOHW/Rn5uXqrqy//dy4EvALqsRnyRJkiSpN58E7hxg+yTbJrk9sD8wr9Ukk2yS5A79402BxzA1d06SJEmSNH9zJnBVdSPwSuAU4DvAiVV1UZIjkuwDkOQRSX4MPBt4b5KL+pc/CFiW5FvA6cBbZ6xeKUmSJEmap/msQklVnQycPOPcYVOPz6EbWjnzdWcCD7mNMUqSJEmSWDuLmEiSJEmSFoAJnCRJkiQ1wgROkiRJkhphAidJkiRJjTCBkyRJkqRGmMBJkiRJUiNM4CRJkiSpESZwkiRJktQIEzhJkiRJaoQJnCRJkiQ1wgROkiRJkhphAidJkiRJjTCBkyRJkqRGmMBJkiRJUiNM4CRJkiSpESZwkiRJktQIEzhJkiRJaoQJnCRJkiQ1wgROkiRJkhphAidJkiRJjTCBkyRJkqRGmMBJkiRJUiNM4CRJkiSpESZwkiRJktQIEzhJkiRJaoQJnCRJkiQ1wgROkiRJkhphAidJkiRJjTCBkyRJkqRGmMBJkiRJUiNM4CRJkiSpESZwkiRJktQIEzhJkiRJasS8Ergkeye5JMmlSQ6Z5frjkpyX5MYkz5px7UVJvtd/vWihApckSZKkdc2cCVySJcBRwFOAHYDnJNlhxtN+CLwYOH7Ga+8GHA48EtgNODzJJrc9bEmSJEla98znDtxuwKVVdXlVXQ+cAOw7/YSquqKqLgBumvHavYDPV9UvquqXwOeBvRcgbkmSJEla58wngdsc+NHU8Y/7c/Mxr9cmOTDJsiTLrrrqqnl+a0mSJElat4xiEZOqOrqqdq2qXTfbbLOhw5EkSZKkUZpPAnclsOXU8Rb9ufm4La+VJEmSJE2ZTwJ3DrB9km2T3B7YH1g6z+9/CvDkJJv0i5c8uT8nSZIkSVpNcyZwVXUj8Eq6xOs7wIlVdVGSI5LsA5DkEUl+DDwbeG+Si/rX/gJ4A10SeA5wRH9OkiRJkrSa1p/Pk6rqZODkGecOm3p8Dt3wyNleeyxw7G2IUZIkSZLESBYxkSRJkiTNzQROkiRJkhphAidJkiRJjTCBkyRJkqRGmMBJkiRJUiNM4CRJkiSpESZwkiRJktQIEzhJkiRJaoQJnCRJkiQ1wgROkiRJkhphAidJkiRJjVh/6ADWBdsc8pkF/55XvPWpC/49JUmSJI2bd+AkSZIkqREmcJIkSZLUCBM4SZIkSWqECZwkSZIkNcIETpIkSZIaYQInSZIkSY0wgZMkSZKkRpjASZIkSVIj3MhbN3PDcUmSJGncvAMnSZIkSY0wgZMkSZKkRpjASZIkSVIjTOAkSZIkqREmcJIkSZLUCBM4SZIkSWqECZwkSZIkNcIETpIkSZIaYQInSZIkSY0wgZMkSZKkRpjASZIkSVIj5pXAJdk7ySVJLk1yyCzX75DkI/31s5Ns05/fJslvk5zff71ngeOXJEmSpHXG+nM9IckS4ChgT+DHwDlJllbVxVNPOwD4ZVXdL8n+wJHAfv21y6rqoQsbtiRJkiSte+ZzB2434NKquryqrgdOAPad8Zx9geP6xx8DnpQkCxemJEmSJGk+CdzmwI+mjn/cn5v1OVV1I/Ar4O79tW2TfDPJGUl+b7YfkOTAJMuSLLvqqqtW6z8gSZIkSeuKNb2IyX8DW1XVLsDBwPFJNp75pKo6uqp2rapdN9tsszUckiRJkiS1aT4J3JXAllPHW/TnZn1OkvWBuwA/r6rrqurnAFV1LnAZcP/bGrQkSZIkrYvmk8CdA2yfZNsktwf2B5bOeM5S4EX942cBX6yqSrJZvwgKSe4LbA9cvjChS5IkSdK6Zc5VKKvqxiSvBE4BlgDHVtVFSY4AllXVUuAY4INJLgV+QZfkATwOOCLJDcBNwMuq6hdr4j8iSZIkSYvdnAkcQFWdDJw849xhU4//F3j2LK87CTjpNsYoSZIkSWLNL2IiSZIkSVogJnCSJEmS1AgTOEmSJElqhAmcJEmSJDXCBE6SJEmSGmECJ0mSJEmNMIGTJEmSpEaYwEmSJElSI0zgJEmSJKkRJnCSJEmS1AgTOEmSJElqxPpDByCtjm0O+cyCf88r3vrUBf+ercQpSZKktpjASeswE01JkqS2mMBJGj0TTUmSpI5z4CRJkiSpESZwkiRJktQIEzhJkiRJaoQJnCRJkiQ1wgROkiRJkhrhKpSStEBaWS1zXY3TlUclSYuBCZwkSbdSK8mwJGnxMIGTJGmRayXRNE5JmpsJnCRJ0iJkoiktTiZwkiRJGkwriabzcjUWJnCSJEnSItBKMqzbxm0EJEmSJKkR3oGTJEmStNa0cqdwrHF6B06SJEmSGmECJ0mSJEmNMIGTJEmSpEaYwEmSJElSI0zgJEmSJKkR80rgkuyd5JIklyY5ZJbrd0jykf762Um2mbp2aH/+kiR7LWDskiRJkrROmTOBS7IEOAp4CrAD8JwkO8x42gHAL6vqfsC/AEf2r90B2B94MLA38O7++0mSJEmSVtN87sDtBlxaVZdX1fXACcC+M56zL3Bc//hjwJOSpD9/QlVdV1XfBy7tv58kSZIkaTWlqlb9hORZwN5V9af98QuAR1bVK6ee8+3+OT/ujy8DHgn8PXBWVf1nf/4Y4LNV9bEZP+NA4MD+8AHAJbf9v7aCTYGfLfD3XBOMc2EZ58JqIc4WYgTjXGjGubCMc+G0ECMY50IzzoW1rsa5dVVtNtuF9Rfwh9xqVXU0cPSa+v5JllXVrmvq+y8U41xYxrmwWoizhRjBOBeacS4s41w4LcQIxrnQjHNhGectzWcI5ZXAllPHW/TnZn1OkvWBuwA/n+drJUmSJEnzMJ8E7hxg+yTbJrk93aIkS2c8Zynwov7xs4AvVjc2cymwf79K5bbA9sA3FiZ0SZIkSVq3zDmEsqpuTPJK4BRgCXBsVV2U5AhgWVUtBY4BPpjkUuAXdEke/fNOBC4GbgReUVW/W0P/l1VZY8MzF5hxLizjXFgtxNlCjGCcC804F5ZxLpwWYgTjXGjGubCMc4Y5FzGRJEmSJI3DvDbyliRJkiQNzwROkiRJkhphAidJkiRJjTCBkyRJkqRGLOoELsnWSfboH2+YZKOhY5opyXpJ/njoODSMJHccOoa5JHlskj/pH2/WbwmiWyHJvZLsk+QPk9xr6Hhmk2RJku8OHcdik2TzJFv1X3OuAK32JNkrybNmOf+sJHsOEdPKtBSr1h1JXppk+/5xkrw/yTVJLkjysKHjm02SPZK8uv969Nr6uYs2gUvyUuBjwHv7U1sA/zVYQCtRVTcBfzt0HHNJsluSR/SPd0hycJI/GDqu+ZgkH2OS5NFJLga+2x/vnOTdA4d1C0kOB14DHNqfuh3wn8NFtKKWKvskf0q3D+Yz6PbLPCvJS4aN6pb6rV4uSbLV0LGsTJJ7JjkmyWf74x2SHDB0XNOSHJrksKlTXwc+DZwK/M0wUc0uyQOTPCnJnWec33uomFYmybMnnbFJXpfk4yN6rx8GnDHL+S8BR6zdUOY0+lj7bagmj4+cce3UtR/RqvV7Hj83yWuTHDb5GjqumZLcP8n7kpya5IuTr6Hj6h0EXNE/fg6wE7AtcDDwjoFimlWSLZOcB7we2Kb/ekuSz/V/C3+6Rn/+Yt1GIMn5wG7A2VW1S3/uwqp6yKCBzSLJW4GfAR8Bfj05X1W/GCyoKX0j/il0+wZ+HngkcDqwJ3BKVb1pwPDmlOSHVTWqxmiSs+ka8Uun/j6/XVU7DhvZivr30S7AeVNxXlBVOw0aWC/Jt4FdquqGJM8F/gp4Ml3Mh1fV7w0a4JQklwCPrqqf98d3B86sqgcMG9ktJfkyXRl+gxXrpH0GC2pKn7i9H/i7qtq5v6P1zTHV7/0H++9V1a/7429W1S5JlgBnVNVjh42wk+TVwCuA7wAPBQ6qqk/2186rqrEkR8Dy+ifJY4E3Av8IHFZVjxw4NJIsq6pdV3JtNPUmtBHr5D3TP17hb3H62lgk+RzwK+Bc4OY9j6vq7YMFNYsk3wLewy3jPHewoHpJzq+qh/aPj6drw7+jPx5VfZRkKfDxqvqPGedfCPwFUFX18DX18xfzMI7rqur6JAD0H/BjzVb36/99xdS5Au47QCyzeRbdB/sdgJ8AW1TVNUn+CTgbGDyBS3LByi4B91ybscxXVf1o8vfZG2KT+7lcX1WVpACS3GnogGa4sapu6B8/DfhAnyCdluRtA8Y1m58D104dX9ufG6PXDx3AHDatqhOTHApQVTcmGd37Z5K89d7Rn/tdkg0HCmk2LwUeXlX/k2Qb4GNJtukbTVn1Swcx+T0/FTi6qj6T5I1DBjRl4yTrV9WN0yeT3A4Y0+8c2oh1VW22Mbbntqiq0d21nsWNVfV/hg5iJW5Kcm/gl8CTWLF9OZa/y4kHzkzeAKrqA0neDKzRZHMxJ3BnJHktsGG68dwvBz41cEyzqqqxzym6sR9W9Zskl1XVNQBV9dskNw0c28Q9gb3o3vTTApy59sOZ04/6sdLVf2AeRNcDPjYnJnkvcNd+WPJLgPcNHNO0lir7S4Gzk3ySrvGxL3BBkoMBquqfhwxuWlXNNrRqTH7d38GcdCzsTtfzPSZ3TnK7SQfD5IM+yR2AjYcMbIb1qup/AKrqiiSPp0vitmacCdyVfZ20J3BkX55jmQ7yceB9SV45def1znTJ+8cHjeyWWoj1jkl2ofv9btg/Tv81tvod4MwkD6mqC4cOZA6fSvJy4BPAdZOTIxn1dRiwDFhCN0LpIoAkvw9cPmRgs5i1fkyyHvDbqvrpGv3hi3gI5XrAAXTDqQKcAvx7jfA/nG4hi4OBrarqwHRzeh5QVZ8eODTg5uF+T6iq3yRZr5+3R5K7AKeP4ZZ2kmOA91fVV2e5dnxVPXeAsFYqyaZ0H5R70P19nko3dGk0d2TS3R7cAnggU++jqvr8oIFNSfI0unmuS4BPVdVL+/O/D/xtVT11yPim9UORV6qq/mFtxbIySa5lxZ7t9MehGw4yisSjn/P0TmBH4NvAZsCzqmpld+LXur4H9l7AK6vqN/25OwHvAn5SVYeu6vVrSz/35eCqOn/q3PrAscDzqmrJULHNpv+83Bu4sKq+13fgPKSqBp8T1ZfbG4E/BX7Qn94KOAZ4/dRogcG1EGuS01d1vaqesLZiWZUkF9LVk+sD29MlGtexvN4cfDjqtCTfn+V0VdUoRn31f5sbVdUvp87diS5n+Z/+eM+h2yJJ/gW4M/AXU50gdwL+hS6BO2iN/vwR5jPrnCQfoRuL/MKq2rH/gDpzMg54aEnuUFXXzXJ+U+Dek96mJJtMv+HGqIUYx2Ss80antVLZT0uyCXD12DqUkvwXXdLxceCEqvrhsBGtXP97fwBdI+mSMTQ4p/Vz3d7E7A3k180cujaUJFvQjbL4ySzXHlNVX+sfj6LuTLId8OOquq6/W7gT3dDpq4eMa1o/RPZ+/eGlVfXbGddHUx+1FOvKDB1jf7d6parqB6u6PjZDl+d8jGE+XD966i3Ai1mxjj8OeG1VXb9Gf/7I2g+32VRPyKzG1hMCyycTz5iw+62q2nno2FbHGN5Qcxk6xiTvZNV/n69ei+HMKclxwLuq6pyhY7kthvy9p1uF7MSq+m4/3OuzdHNKbwSeW1WnDRHXyvR31p8B7A9sQLe40gkjGV4DQJJnzHL6V3R3ZdbosJXVtRgayDB83TkVx/nArnQrvp0MfBJ4cFU1sSoyjKcs56OFWMcSYz+U+6KqurY/3hh4UFWdPWxkq2cs5bkqGdEiNjPq+MsmIy6mrq+ROn4xzoF72tAB3ArX938Ak/kc2zE1LrkhY5wvMdPQMS4b+OevrkcCz0vyA7rVCEc5JGQehvy97we8oX/8Irr5HJsB96frqRtVAldVvwLe3yfv+wP/RpfIjWaOHt3w+EfRrYYL8Hi6UQzbJjmiqj44VGAz9QnbqubEHEm3uu/YDV13TtxU3aI1zwDeWVXvTPLNoYNaTWMpy/loIdaxxPh/WHHhiv+Z5VwLxlKeqzKau09D1fGLLoFr7VZ17++BzwFbJvkQ8BhgdHuXzcNo3lCrMGiMVXXckD//Vthr6AAWyJC/9+unhkruBXy4XxToOxnhhs794jrPAX4P+CrwR1X1lWGjuoX16Xq2/x9AknsCH6DrcPgyMJoEbh5aaCzBeOr3G5I8B3gh8If9udsNGM+tMZaynI8WYh1LjJkeFl9VN42xjp+HsZTnYrFG6vgW/7Dmpb+V/U7gQcDt6RY5+PVYJuFPq6pTk5wL7E73iz6oqn42cFhag5JsRrdB9g50dzcAqKonDhbULCYdIknuwVScWi3XJdkR+H/AE4C/nrp2x2FCml2SK4CrgROAA+mGeU4WDaGqzhsqthm2nCRvvZ/2536RZFRz4ebBxtLq+RPgZcCbqur7SbalrYRdi9fl6fZVnCzR/3LGt3Li6PWLEO5eVataQfyKtRTOQlgjdfyiTeDoVvraH/go3Xj5F9INWRqdJF+oqicBn5nlXEta6EkeS4wfoptb9FS6xsiLgKsGjWgWSfYB3g7ch66RvDXddgcPHjKuW+GKAX/2QcDH6IZN/ktVfR8gyR8AYxv6dQXdh81eLF95dKKAsXQwfCnJp+nqd4Bn9ufuRJeAauGNou6sqouBV08df59uiNIoLMLG5xVD/vDGyvNldEPOX0dXX36BriNsNFooz/7O5VHASue4VdVs86DXKYtuEZOJqYVBLpjM1xnTpEeAJBvQ9cCfTjeHY/IBuTHwuap64EChzWqu1b+S3G3ohQ5aiLGP49yqeviMv89zquoRQ8c2Lcm36Brtp1XVLkmeADy/qg4YODRgpYtZ3KyqxrKf0aIy9MIbSUK30Mpj+1O/BO5ZVa8YKqbZzKexlOTjY2iMjL3uTHJiVf3xyhYqG9O83LG1NWbTUt3ZSHkuoXu/PG/oWObSSHn+E/B14ONjW615da2pOn4x34H7TZLbA+cneRvw34xns8+JPwP+gu7uxrksT+CuobuDODYnAbsmuR9wNN3qX8cDfwCj2QSyhRgBJsO8/jvJU4H/D7jbgPGszA1V9fMk66XbA/D0JP86dFBTJnNg7gE8Gvhif/wEug3cx9QIuTtwOF3SUXTzy46oEe39txoGXXijqirJ5XTDzp8NfJ/uvT8qjfUkj73unOyp1MJCZV9I8kzG3fhspu6kgfKsqt8l2TrJ7df08vELYPTlSdc+Phj4XZLfwrj2IoX5d4KsqTp+Md+B25puyNftgL8E7gK8u6ouHTSwWSR5VVW9c+g45jJZWjbJ3wD/O1n9a0w9OS3ECEw2oP4KsCXdXM2NgX+oqqWDBjZDktOAp9PtdbIp3XvqEVX16CHjminJqcCLquq/++N7A/9RVaNZhCXJ5+kW2PjP/tTzgMdX1R7DRXXrDPWeSnJ/ugVWngP8jG4Y8l9X1Sr3YRpSKz3JrdSdLUhyLXAn4HfAKBufE43UnU2UZ5IP0K27sJRu1WYAqmpMK/g2U55jl+T9/cNZO0Gqao12Ni3aO3C1fDXK3wL/MGQs8/CTJBtV1bVJXke35OwbR7RgwMRk9a8XMd7Vv1qIkar6dP/wV3Rv9lFJslV1mzjvS/ce+ku6hOMuwBFDxrYSW04aIL3/R7eh5pjcu6reMHX8xiT7DRbNbTNUIvJduo6Pp00645L85UCxzNfoe5J7o647+0bn9N9d+uPRlWdVbTR0DKth9HVnQ+V5Wf+1HjDamFsoz36Y/POAbavqDUm2pPsM/cbAod2sqv4Ebu4E2WFmJ8ia/vmLNoHr73C8gW7RhfUZYSU/5fVV9dEkjwX2AP6RbhWjRw4b1i20sPpXCzHSx/Uqus1ob34fVtU+Q8U0w38BD6uqXyc5qaqeSbdn2Vh9IckpwIf74/0Y2f5qwKlJ9gdO7I+fBZwyYDwtmmwwfnqSz9GtljmKxTVWpoXGUm/sdecXgHvRDe07oe9gGqUWGp9TRl93tlKeVTX2mwVAM+X5buAmujn4b6DbU+8oYFTrBPQG6QRZzEMoL6X7sL9wzMNWYPlwpCRvoYv3+LEPXUmyCd0f7QVDx7IyY46xXxzkGLrNH2+anK+qMwYLasr039/Y/xYnkvwR8Lj+8MtV9Ykh45mYunMQlg9bgW5rk/8ZaafSKg298Ea61Sb3pRtK+US6PeA+UVWnDhXTyjTSWFrBWOvOJHdheRK/Ad0Q2hNGMD9vBUn+D33js6oe1JfnqTWyRaomxlp3TrRSnum2B/pbulWaR7s9UAvlOTWke7ot8q2q2nno2GZK8i5ge1bsBLm0ql61Jn/uor0DB/wI+PbYk7felUneC+wJHJnkDoxvwRWSfAnYh+7v5lzgp0m+VlUHDxrYlBZi7P1vVf3b0EGsQq3k8ZidB1xbVaclueNkWPLQQTV0B+ZmSS6gu7v1kaq6bOb1IZO3/uf/mm6BjeP7xsez6fZVHF0CRyM9yS3UnVX1K+D9SY6jS+L+ja6hPKo5RsAjJ41PgKr6Zb+o2liNsu6c0kp5TrYHehoj3h6INsrzhnQrexbcnBzftOqXDKOqXjmjE+TotdEJspgTuL8FTk5yBnDd5OTYJpP2/hjYG/inqrq6Hz/7NwPHNJu7VNU1Sf6Ubrncw/uG3pi0ECPAO5IcTtfgnP77HMu8x52TXEN312jD/jGMdChykpfS7bdzN2A7YHPgPcDgeykmeWBVfTf9Ztgzjeh3Pu0P6XoRT0xyE12j5MQxDlurql/SrZp49NCxrEQLjSVooO5M8mi6u66/R7eK6x9V1VeGjWpWzTQ+x1x3TmmlPO9eVcckOagfTXNGknOGDmoWLZTnvwGfAO6R5E10Uw5eP2xIq7TWO0EWcwL3Jrqezg2AMX5Y3qyqfgN8PMk9kkzGzX53yJhWYv0+ufxj4O+GDmYlWogR4CHAC+h65ScV52g2Sq6qJfN5XpJN+gb00F4B7AacDVBV30tyj2FDutnBdA2kt0+dm76rOYrf+bTqFoF6G/C2JNvTfXAeSTfsU6unhcYSjLzuTHIF3SbtJ9C9n27szz8MRtcR0lLjc8x150Qr5dnK9kCjL8+q+lCSc+k6EgI8vaq+M3BYsxqqE2QxJ3D3qaodhw5iPpLsQ9e4uw/dMu1b0SVwDx4yrlkcQbfowler6pwk9wW+N3BMM7UQI3RDvu5b498vZi5foFs1dWjXVdX13XQjSLI+4xn6+e9J7lVVTwBI8iLgmcAVwN8PGNcqpduKZb/+63d0oxq0+kbfWOqNve68gu49vRfwZFZcvGY0nV/QVuOTcdedQFPl+cZ+nuZfsXx7oNGtkttCeSb5YFW9gKmbGVPnxmaQTpDFvIjJ24DTxjipfaZ+QYsn0sW7S5InAM+vqgMGDk1rSJL/Ag6sqp8OHcttMZYFTvr3+9XAC+lW93w5cHFVDX4nIcl5wB5V9Yskj6O7g/Aq4KHAg6rqWUPGN5skZ9MtIf9Runlwlw8cUtOSPJDljaUvjK2xtJgk2bOqBttkvo/hFg3NsTY+x1x3Toy9PJNsQDfn7X50C5MdU1U3DhvVyo29PGH5IiZTx0voFvnbYcCwZpXk7Kp6ZJYvSLg+cF5V7bQmf+5ivgP358BfJ7mO7rb2KOfu9G6oqp8nWS/JelV1epJ/HTqomfpK6gBuucLSSwYLaoYWYuzdFfhuPz5+eg7cWLYRmK+x9AAdQvd7v5Bu362Tq+p9w4Z0syVTq+TtRzfB+STgpCTnDxfWKr2wqi4ZOojFoJWe5IbqzrkcCQyawDFj9Ezf+Hz4QLHMZcx158TYy/M4unbmV4CnADsABw0a0aqNtjyTHAq8llvOvb8eGNvf5cQZSSYx70nXCfKpNf1DF20CN9fKb0keXFUXra145nB1kjsDXwY+lOSnwK8Hjmk2H6RrhOxFN9zmecDYepJbiBHg8KEDWGT+vqoOo6/gkyxJ8qGqet7AcQEsSbJ+3yP7JLqx8hOjqoOTHDz1+Kkzr490EaixG21jaYZW6s65DLYvYKONz9HWnQ2V5w5V9RCAJMcAo9wipIXyrKq3AG9J8paqOnToeOZpkE6Q0S1VvxaNaYPSfYHf0o2V/hxwGd0qcGNzv6p6PfDrqjoOeCrj22y8hRgn+71dAdyuf3wO3SpGrRnLJspb9h9O9Cv8ncR45u98mK6H7pN07/OvACS5H/CrIQObxUb91650oxg2779exjjmOjYjyaHp9gDcKck1/de1dPOclw4c3myaqDvnYbBRAVX1lr7z+B+rauP+a6OquntVHTJUXHMYbd3ZUHlOFi9hzEMnGypP6OaUrSDJF4YIZB7+vqreV1XP7qdEHJvkQ2v6h46q93ctG0vDc7Kn0cRxgwUyt0kldXWSHYGfAGNbraqFGFtZuhlYvqkvU/XF1IpvY4n3JXR3rw8FngB8tqr+ZeCYAKiqN/UfPPem2yx10sBcj27OyWhU1T8AJPky8LDJMshJ/h74zIChNafBnuQm6s5GzNr4rKqx1JfTRlt3Thl7ee48447W5A7XWKfujLY8+6HcdwI27dsek7b6xnTtpDHaMsmhVfWWvhPkROD8Nf1D1+UEbvC5O31v7HQc6Y/H+qY/un9DvZ6uB/nOwGHDhnQLLcQIbSzdTJI3AC+muys8+Vu9ecW3qbldg8iKe6u9A3gv8DW6O14Pq5EsLV5VZ81y7v8OEcs83ZNuWM3E9f05rb7RNpZmaKXunMsVQ/3glhqfLdSdrZRnzXPbnaE1Up5/BvwF3ars03+D1wDvGiKgeRikE2TRrkI5l5kr3AwUw38B9wI+DpxQI9wkV2vGUKsWra4klwAPqZFud5Dk9FVcrqoazdLiLUnyd3T7gX2iP/V0uo283zxYUI2Zaix9EXg8KzaWPldVDxwotKal21z8BLrVUS8bOp5pSQ5ieePz/5u6dA3wvqoaTQO0hbqzpfJsQUvlmeRVVfXOoeNYlRmdILdjeSfIMbDm96ZclxO4s6pq9xHEcRfgGcD+dCt/fYQumRv0zsa06YUNZjOGhQ1aiHFaC0s3AyQ5CfjzGvF2B0nWA55dVR8ZOpbFJMnDgcf2h1+uqm8OGU9rWmksNVh3Tu9PeBPdZ+aJY+oAbaHxCe3Una2UZytaKM8kd6JbF2KrqjowyfbAA6rq0wOHdrOhO0EWbQKX5ICqOmbqeAnwuskcj7HpK9L96TZ9ffOYPjSTrHLFxDGUaQsxTut/3wewfEPaU4B/r5G9IZPsCnwS+DYj3u4gybKq2nXoOBabfljv9JLyo2kkt2LsjaXW6s5pfaPu9cDzxjSMrYXG50QLdWdL5dmCFsozyUeAc+m2tNkxyR2BM6vqocNGtqIhO0EWcwJ3PN1eWwfQLRTxH8AZVfXXA4Z1C0keDTwH+D3gq3TDQr4ybFRaUyZzX5IcWVWvGTqeuSS5iG5YwIV0vd3AzatojkaStwI/o+uNv3lRoDHdyW5Jkn2At9PdPfopsBXw3ap68CpfqFtoobHUmhl34X5H97n59mGjWq6Vxie0UXe2VJ4taKE8Jx0Lk2km/blvVdXOQ8c201CdIIt2EZOqem6S/eganr8GnltVXxs4rBUkuYJuGN0JdCsS3tiffxis+fGzqyvJccBBVXV1f7wJ8PYa0UavDcR47z5p3yfJCcxYDXVsv3PgN1X1b0MHMQ/79f++YupcAfcdIJbF4A3A7sBp/RzNJwDPHzimVh1L11h6dH98JfBRYFQJXAN1J9DNH6abb/JRup7vywcOaTbbVdV+SZ4DUFW/STKala9naKHubKk8W9BCeV6fZEP6xdOSbMfUKKCROS3JX7OWO0EWbQLX93IeRLenyYOAF/SZ/G+GjWwFV9D9ce7F8qF0Ezev9DciO00+3AGq6pdJdhkwntmMPcbD6Ib8bAHMHCY7xt/5V5K8hW5VuukhlKNKNKtq26FjWGRuqKqfJ1kvyXpVdXqSfx06qEa10FiC8dedEy+sqkuGDmIOzTQ+G6k7mynPRrRQnofT7Yu8Zbo91R5DtyL2GA3SCbJoEzjgU8Arq+q0/sPyYLrNkkczBKiqHj+f5yXZs6o+v4bDmY/1kmxSVb8ESHI3xvc3NOoYq+pjwMeSvL6q3rCy5yV5cFVdtBZDW5lJA256wZ8xJpqk27tqB1acs/WB4SJq2tVJ7gx8mW555J8y1bOo1dJCYwlGXndOL7aS5Kkzr49p3jhtNT5bqDubKs8GjL48q+rzSc6ja3uEbnTAzwYOa1ZDdYIs5jlwG1fVNTPO3b/GvffSrDKCLQ/6OF4I/B3dJoUAzwbeVFUfHC6qFbUQ43yM4XfeL/zz6hrfpq630C/E8Hi6RsjJwFOAr1bVs4aMq1X9vK3f0m02/jzgLsCHqurngwbWoCR7Aq+j+9s8lb6xVFVfGjKumcZed04ttvIA4BF0owIA/hD4RlWNaohvkruzvPF51lgbn63Una2UZyvGWp5ZcWn+Wxjb6J+JITpBFnMCd0/gzcDmVbV3kh2AR02vTNmK6UmcQ+vLcXL35YtVdfGQ8cymhRjnMpbfeZJvVNUtNiIemyQXAjsD36yqnfv3/39W1Z4Dh7YoJfl6VT1q6DhaMdbG0kwt1J1Jvgw8taqu7Y83Aj5TVY8bNrI2G59jrjtbLM8xa6E808D+hDMN1QkymuERa8B/AO+n61EE+L90EwybS+Doh94MJd2GtC8D7ke3KMx7qurGIWOaqYUYV9NYela+luRd3HJy7uAV/Qy/raqbktyYZGO6lRO3HDqoRWyDuZ+ybpulsfTf/b9bJdlqLO+hBuvOewLXTx1f358bg1WthDnKoeeMu+5ssTzHbPTlWVVPgK5eqqr/nb7W11Vj9CyWd4L8yaQTZE3/0MWcwG1aVScmORSgqm5M8ruhg2rUccANwFfoehYeRLdB7Zi0EGOLHtr/e8TUuVFU9DMsS3JX4H10K/79D/D1QSNa3MbSwTBmo28s9VqrOz8AfCPJJ/rjp9P9HwbXaONztHVno+U5Wo2V55nAzE6w2c6NwSCdIIs5gft1P2xlMnF8d+BXw4Z0q10x8M/foaoeApDkGOAbA8czmxZiXB3Xz/2UNW9S4Y9dVb28f/ieJJ8DNq6qC4aMSeu2hhpLTdWdVfWm/j3+2P7Un1TVN4eMaRbNND4bqTubKc9GjLY8k9wL2BzYsF8Fd7Ji78bAHQcLbNUG6QRZzAncwXSTnLdL8jVgM7rbnKOT5AK6veA+UlWXzbxeVc9Y+1Gt4IbJg/5O5pCxrEwLMd4syQHT8zH7BUNeV1X/AFBVu6/0xWtRkrvQrVg1mV9yBnBEVY2uMyTJM+gadQV8FRhbI2QxGfcbbFxG21jqNVV3AlTVuUl+RD+Utx+S+sOBw2q18TnaurPV8hyrRspzL7oVMbegG8UwifFa4LUDxbRKQ3WCLLpFTJI8AvhRVf0kyfrAnwHPBC4GDlvTG+vdGkm2pttHYj/gJrr5RieO4QMJoB96Opn/FGBD4Df946qqjYeKbaKFGKclOR64K3AAcDe6OZtnVNVfDxjWLSQ5Cfg2y4covQDYeQSdCitI8m66OTwf7k/tB1xWVa9Y+au0Kn29tH2/FcuGwPpTC0fsWFXfHjbCcZtqLP0n8FxWbCy9p6oeOFRs0xqsO/eha9jdh26o0lbAd6tq8C2CkryIrvG5K922RdONz/+oqo8PFNpKjbnubLE8x6yl8kzyzKo6aeg45mtmJ0hVfWKOl9z2n7kIE7jzgD2q6hdJHkd3Z+tVdHN5HjS2pXFnSrcB+euB51XVkqHj0ZqTZD/gKLrG03Or6msDh3QLSc6vqofOdW5oSb5L9/6eDJleD7ioqh40bGRtSvJS4EDgblW1XV8vvaeqnjRwaM1oqbHUkiTfops/eFpV7ZLkCcDzq+qAgUO7WUuNzxbqzpbKswUtlGeSg+gWIryWbmjiw4BDqurUQQObxVCdIOutyW8+kCVTd9n2A46uqpOq6vV0BTxKSbZO8rd0CecDgb8dOCStQX2D+CDgJOAHwAuSjGUIw7TfJpnMNSHJY+j2BxubS+l64ie27M/p1nkF3X5l1wBU1feAewwaUWOq6rh+HtyLq+qJVfWE/msfk7fb5Ibq9iNcL8l6VXU6XZI8Jlsk2Tidf09yXpInDx3USrRQd7ZUni1ooTxfUt1ezk8G7k43+uetw4a0Uk8E9qqq91fV+4E/YC0sUrUoE7h+6CTAk4AvTl0b5Zy/JGcDnwCWAM+uqt2qalUrmKl9n6Ib0vtnwO8D36PrpR+bPweOSnJFkiuAd9ENSx6FJJ9KshTYCPhOki+l20fmO/053TrXVdXNC+n0deriGq6x9rTQWGrJ1UnuDHwZ+FCSdzC1xclIjL7x2VjdOfrybEwL5TkZsfAHwAeq6qKpc2MzSCfIKBOa2+jDwBlJfkZ3p+ArAEnux3hXoXxhVV0ydBBaq3brK1D6oStvT/KpgWOazXeAtwHb0c3Z+xXdst2jmOQO/NPQASxSZyR5Ld1k9z2Bl9N1Omj1vaSq3pFkL5Y3lj4IjG4oUCP2pfts/0vgecBdWHGbkzG4ReMz41sdpqW6s4XybEkL5XluklOBbYFDk2xEt0bEaPRttmJ5J8g3+uNHshZW8110c+Dg5i0D7g2cWlW/7s/dH7jzWDZPBUhy8KquV9U/r61YtHal2+jxzcDmVbV3kh2AR02vTDkG/YpKVwPnATfvozjWO8Tp9mC5uWNqjIsWtaCfB3MAXQ9tgFOAf6/F+IGxhiW5oKp26u8UfamqPpHkm1W1y9CxLUZJvl5Vjxo4hvfTLWCzLd0Gv0vofvcPHzKuVRlz3dlieY7Z2MuzTya3oFs9/vKqujrdtmCbj2mLiyS/v6rrVXXGGv35fh4PJ8nh/cMHAI+g2/YA4A+Bb1TV8wcJTGtcks/STdD9u6rauR+i9s3JfkxjkeTbVbXj0HHMJcmBdL3w/0vXSzdZQe++gwamdd7YG0uLzdDJcSuNz4mx152tlefYtVKeSS4cW3toLmu7E8QEbgSSfBl46tQS3RsBn6mqx636lWpVknOq6hHTjY2Mc3XHo4F3VtWFQ8eyKkm+R3cH82dDx9KyJBeyirluVbXTWgynea00lhaTJOdV1aB77LXU+Gyh7mypPFvQQnkmOQ54V1WNcW2AFQzVCbIY58C16J7A9VPH1/fntHj9um/ITZZu3p1xztF8LPDiJN8HrmN5xTS2hvxldHtX6bZ52tABLCZVVUlOnm4s9Sso/nzAsLTmnZfkES00Pmmj7mypPFvQQnk+Enhekh/QLVI01rYHwN8AO67tThATuHH4APCNJJON/57O8o2TtTgdTDdkdrskX6ProR/jHoVPGTqAeToUOLNf0fW6ycmqevVwIbWnqn4AkOTIqnrN9LUkRwKvmfWFWpUWGkuLyRgWY2ip8dlC3dlSebaghfLca+gAVsMgnSAOoRyJJA+nu9sB8OWq+uaQ8WjNSPII4EdV9ZN+3tufAc8ELqbbVmA0E8db0q/+9FXgQqZWqqoqO0JuhdmGoU0W4xgqplal2yj5fnT7PY61sdSUJFsD21fVaUk2BNafmoKwY1V9ewTx3cKkg2RMWqg7WyrPFoy5PJNsXFXXJLnbbNfH2EZKsgvdmgZrtRPEBG5EktwD2GByXFU/HDAcrQFJzgP2qKpfJHkc3cbtrwIeCjyoqsZ4F270hl64YLFI8ud0Wwbcl65XcWIj4GsurLT6xtxYalGSlwIHAnerqu2SbA+8p6qeNHBorTY+R1t3tlieY9ZCeSb5dFU9rZ+2Uax4R300i+tMG6oTxARuBJLsA7wduA/wU7oNAb9bVQ8eNDAtuCTfqqqd+8dHAVdV1d/3x6NbxKQVSd4MXEG3V9l0D9jgH0gtSXIXYBPgLcAhU5eutSxXTwuNpRYlOR/YDTh7agGoUSzK0Gjjc7R1Z4vlOWYtlGeSx1bVV5NsUFX/O3Q88zFUJ4gJ3Agk+RbwROC0qtolyROA51fVAQOHpgWW5NvAQ6vqxn5o1YFV9eXJtWpgyf4x6j+QZhrFB1JLTDoWTguNpRYlObuqHjlpNPVD0c8bw5DURhufo607WyzPMWuhPJOcW1UPH8NqsvM1VCeIi5iMww1V9fMk6yVZr6pOT/KvQwelNeLDwBlJfgb8FvgKQJL7Mc5VKJtQVdsOHcMicTzdSpTnMkvSQTe0UvPz1v7fB421sdSoM5K8FtgwyZ50Q34/NXBME+8AHg6cCTTR+Bx53dlceY5cC+V5Q7990RZJ/m3mxZEtrjPxnP7fQ6fOrfHPS+/AjUCS0+hWnnwLsCndMMpHVNWjh4xLa0a/ZcC9gVOr6tf9ufsDd66q8wYNrjFJ/raq3tY/fnZVfXTq2pur6rXDRad1WYs9yS1Ish5wAPBkug6GU4B/rxE0ZpKcBVxA93l+wszrY2p8tlB3tlSeLWihPJNsCuwBHAkcNvP6mBbXGZoJ3AgkuRPd3Zj1gOcBdwE+VN1+QZJWYrpxPLOhbMP51kvyhZmLQsx2TivXQmNJC6ulxmcLdWdL5dmClsozyc5V9a1VXD+0qt6yNmOaJYZBO0EcQjkCk7swdKvX3OINlOTrVfWotRuV1ISs5PFsx5pDkg2AOwKbJtmE5WW4MbD5YIG16Wl0jaW96Iak6jZIciHdsKRZjWEOXHUb+Z6Q5Dtjb3zSQN3ZWHmOXkvluar4es+mG7U2pP2Bt/WPDwU+OnVtb8AETsu3FpC0glrJ49mONbc/A/6CbkXcc1nekLsGeNdAMTWppcZSI542dADz1Ujjs5m6s5HybMYiKc8xdDIM2gliAteGUVWm0ojsnOQauspyw/4x/bEdH6upqt4BvCPJq6rqnSt7XpI9q+rzazG0Zi2SxtLgqt83L8mRVfWa6WtJjgReM+sLx2kMjc/FVHeOoTwXkxbKcwzt4kE7QUzgJDWrqpYMHcNitKrkrXckYAK3MFpoLI3JntwyWXvKLOfGbPDG5yKrOwcvz0WmhfIcQ705aCeICVwbxvCHKkkT1kkLp4XG0uCS/DndlgH3TXLB1KWNgK8NE9Wt5vtnYVmeC2vw8kzymKr62irOfXSWl61VQ3eCmMCNRJKtge2r6rQkGwLrV9W1/eUXDBiaJM1k0rFwBm8sNeJ44LN0w00PmTp/bYMbzA/e+FxkLM+FNYbyfCe33Kvu5nNV9ea1HtHIuI3ACCR5KXAgcLeq2i7J9sB7XLJb0hiNZZnxxSDJa22MzC3JxlV1TZK7zXZ9TElcks2AlwLbMNVRXlUvGSqmllmeC2vM5ZnkUcCj6RbT+pepSxsDf1RVOw8R1xh5B24cXgHsBpwNUFXfS3KPYUOSpJW6YugAWjFXY8nkbd6Op1uJ8ly6O8DTdy4LuO8QQa3EJ4GvAKcBvxs4lsXA8lxYYy7P2wN3pqsrN5o6fw3wrEEiGinvwI1AkrOr6pFJvllVuyRZHzhvDPvaSFr3JLkj8FfAVlX10n5UwAOq6tMDh9acJGfSNZbOZaqxVFUnDRaU1qgk51fVQ4eOY7GwPBdWC+WZZOuq+kGSO1bVb4aOZ4zWGzoAAXBGktfSrWKzJ934408NHJOkddf7geuAR/XHVwJvHC6cpt2xql5TVSdW1UmTr6GDalWSL8zn3MA+neQPhg5iEbE8F1YL5XmfJBcD3wVIsnOSdw8c06h4B24EkqwHHAA8mW5YyCnAv5e/HEkDSLKsqnadjAroz33L+QerL8kbgTOr6uShY2lZkg2AOwKnA49n+RDKjYHPVdUDBwrtFpJcC9yJrhPkBrpYq6o2HjSwRlmeC6uF8kxyNt2QyaVTn0Hfrqodh41sPJwDNwJVdRPwvv5LkoZ2fb8abgEk2Y7uw16r7yDgtUlG21hqxJ/RLWxwH7rhqJME7hrgXQPFNKuq2mjuZ2m+LM+F1Up5VtWPkhUW6R3bfL1BmcANKMmFrGI5bufASRrI4cDngC2TfAh4DPDiQSNqVCuNpbGrqncA70jyqlVtNJ9kz6oaZJP5JA+squ8mmXWF1qo6b23H1DLLc2E1Vp4/SvJooJLcjq4j7DsDxzQqDqEcUL/320pV1Q/WViySNC3J3YHd6e50nFVVPxs4pKY01lhaNIbc4iLJ0VV1YJLTZ7lcVfXEtR5UwyzPhdVSeSbZFHgHsAfdZ9CpwKvHtF3I0EzgRiDJkVX1mrnOSdLakmQnbrn0/ccHC6gxLTWWFpPpeZuS2pTkKVX12RnnXlZV7xkqprExgRuB2XoMk1zgEEpJQ0hyLLATcBFwU3+6xrDRq7QqY9hkvl9w5eXAY+mmSXwFeE9V/e+QcbXK8lxYLZRnv/3K66rqi/3x3wBPrKqnDBvZeJjADSjJn9O9ie4LXDZ1aSPga1X1/EECk7ROS3JxVe0wdByLQQuNpcVkJAncicC1wH/2p54L3LWqnj1cVO2yPBdWC+XZD6H8NPA3wN7AA4HnVNX1gwY2IiZwA0pyF2AT4C3AIVOXrnWcr6ShJDkGeHtVXTx0LK1robG0mCT5eFU9Y+AYbtEBYqfIrWd5LqxWyjPJPYDT6FadfYlba63IVSiHVVV1RZJXzLyQ5G4mcZIG8gHg60l+Qrd9wGTpe4d1r74dZzSMTu83qNWtkOSOwF8BW1XVS5NsDzygqj4NMHTy1jsvye5VdRZAkkcCywaOqWWW58IabXn2e9QV/WcOcHu6UWrPSuL2K1NM4IZ1PPA0ut6FyR/sRNH90UrS2nYM8ALgQpbPgdOtM9rGUqPeT/eZ+aj++Ergo3TDrQY1tTXQ7YAzk/ywP94a+O6QsbXI8lxYLZSn267Mn0MoJUkrSPL1qnrU3M/UysxoLD0AWKGxNLbhSq1Isqyqdp1ebTLJt6pq5xHE5tZAC8jyXFgtlWeSPwK+WFW/6o/vCjy+qv5ryLjGxDtwI5DkC1X1pLnOSdJa8s0kxwOfohtCCbiNwGp62tABLFLXJ9mQLhkmyXZM/Y0OaWYDuJ/Ds8FA4TTP8lxYjZXn4VX1iclBVV2d5HDgv4YLaVxM4AbUr052R2DTJJuwfAjlxsDmgwUmaV23IV2j+MlT5wowgZunxhpLLTkc+BywZZIPAY8BXjxoRDMk2Qd4O3Af4Kd0d12/Azx4yLhaZXkurEbKc71ZzpmzTHEI5YCSHAT8Bd2b6EqWJ3DXAO+rqncNFJokaQGsrLFUVWNqLDUlyd2B3ek+M8+qqp8NHNIKknwLeCJwWlXtkuQJwPOr6oCBQ2uS5bmwWijPfi/Sq4Gj+lOvAO5WVS8eKqaxmS3D1VpSVe+oqm2Bv66q+1bVtv3XztPJW5I9BwxT0jomyQZJXpHk3UmOnXwNHVej3kCXbPzfvr5/EnDWsCE1b3NgCd0KdY9LMoaVJ6fdUFU/B9ZLsl5VnQ7sOnRQDbM8F1YL5fkq4HrgI/3XdXRJnHrejhyBqnrnHE85Evj82ohFkoAP0q1KthdwBPA8uiE2Wn03VNXPk9zcWEryr0MH1aq+I2En4CKWr5A6tuG9Vye5M/Bl4ENJfgr8euCYWmZ5LqzRl2dV/ZoV90fWDA6hbMD0aluStKZN6pwkF1TVTkluB3ylqnYfOrbWJDkNeDrwFmBTumGUj6iqRw8ZV6vGuOHwRJL7AfcEzgd+SzfK6Xl0w2Y/U1XnDhddeyzPhdVCeSZ5V1W9Msmn6BcqmlZV+wwQ1iiZwDUgyXlV9bCh45C0bkjyjaraLcmXgZcDPwG+UVXuTTlPLTSWWpTkGODtVTW6zdCTfBo4tKounHH+IcCbq+oPh4msTZbnwmqhPJNcU1UbJ/n92a5X1RlrO6axcgilJGmmo/uVcV8HLAXuDLx+2JCa8690jaXJ0KSbgOMmjSVg8MZSoz4AfD3JT+jmxQSoqtpp2LAAuOfMxjFAVV2YZJsB4mmd5bmwWijPy8BEbT5M4NpwxdABSFqnfBB4JrANcFx/7p6DRdOmFhpLLToGeAFwIcvnwI3FXVdxbcO1FcQictdVXLM8V99dV3FtLOW5WZKDV3axqv55bQYzZq5COQJJ7pjk9Une1x9vn+TmTWCramwrbEla3D4J7AvcCPxP/zWqSe4NuOsqro2lsdSiq6pqaVV9v6p+MPkaOqjesiQvnXkyyZ8CDpldfZbnwmqhPJfQjfjYaCVf6jkHbgSSfITuzfPCqtoxyR2BM6vqocNGJmldlOTbVbXj0HG0LMmHgS9W1ftmnP9TYM+q2m+YyNqW5N10yfGn6IZQAlBVg69CmeSewCfolj+fNIh3pdvu4I+q6idDxdYiy3NhtVCervkwfyZwI5BkWVXtOr3aZJJvVdXOQ8cmad2T5GjgnbMNAdT8tNBYalGS989yuqrqJWs9mJXoN0aedIBcVFVfHDKe1lmeC2vM5emq6/NnAjcCSc6k29z1a1X1sCTbAR+uqt0GDk3SOijJxcD9gO8zvoUimjLmxpIkjUmSu1XVL4aOowUmcCOQZE+61d52AE4FHgO8uKq+NGRcktZNSbae7fyI5hppHZVkA+AA4MHABpPzY7oDJ0lrmgncSCS5O7A7XU/3WVX1s4FDkiRpVJJ8FPgu8FzgCLq99b5TVQcNGpgkrUUmcCORZCe6Jbtv3tphDJOyJUkai8kcmSQXVNVOSW4HfKWqdh86NklaW9wHbgSSHAvsBFzE8n1tCjCBkyRpuRv6f69OsiPwE+AeA8YjSWudCdw47F5VOwwdhCRJI3d0kk3o5o0vpdsz6vXDhiRJa5cJ3Dh8PckOVXXx0IFIkjRiHwSeSTfl4Lj+3D0Hi0aSBmACNw4foEvifoJLdkuStDKfBH5Ft7fedXM8V5IWJRcxGYEklwIHAxeyfA6cS3ZLkjQlyberase5nylJi5d34MbhqqpaOnQQkiSN3JlJHlJVFw4diCQNxTtwI5Dk3cBdgU8xNSTEbQQkSVouycXA/YDv45QDSeso78CNw4Z0H0RPnjrnNgKSJK3oKUMHIElD8w6cJEmSJDXCO3AjkGQD4ADgwcAGk/NV9ZLBgpIkSZI0OusNHYCAbl+bewF7AWcAWwDXDhqRJEmSpNFxCOUIJPlmVe2S5IKq2inJ7YCvVNXuQ8cmSZIkaTy8AzcON/T/Xp1kR+AuwD0GjEeSJEnSCDkHbhyOTrIJ8DpgKXBn4PXDhiRJkiRpbBxCOQJJ7gA8E9gGuF1/uqrqiMGCkiRJkjQ63oEbh08CvwLOZWojb0mSJEma5h24EUjy7araceg4JEmSJI2bi5iMw5lJHjJ0EJIkSZLGzTtwI5DkYuB+wPfphlCGbg7cToMGJkmSJGlUTOBGIMnWs52vqh+s7VgkSZIkjZcJnCRJkiQ1wjlwkiRJktQIEzhJkiRJaoQJnCRJkiQ1wgROkiRJkhrx/wOyF8Fsd5VOsQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1080x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "feat_importances.nlargest(20).plot(kind = 'bar', figsize = (15,6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}