examples/notebooks/RandomForestTest.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from loaders.Loaders import CSVLoader\n",
    "from compoundFeaturization.rdkitFingerprints import MorganFingerprint\n",
    "from featureSelection.baseFeatureSelector import LowVarianceFS\n",
    "from splitters.splitters import SingletaskStratifiedSplitter\n",
    "from models.sklearnModels import SklearnModel\n",
    "from metrics.Metrics import Metric\n",
    "from metrics.metricsFunctions import roc_auc_score, precision_score, accuracy_score, confusion_matrix, classification_report\n",
    "from parameterOptimization.HyperparameterOpt import HyperparamOpt_CV, HyperparamOpt_Valid\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mols_shape:  23290\n",
      "Features_shape:  X not defined!\n",
      "Labels_shape:  (23290,)\n"
     ]
    }
   ],
   "source": [
    "#Load Dataset\n",
    "dataset = CSVLoader(dataset_path='data/preprocessed_dataset_wfoodb.csv', \n",
    "                    mols_field='Smiles', \n",
    "                    labels_fields='Class', \n",
    "                    id_field='ID')#, shard_size=4000)\n",
    "dataset = dataset.create_dataset()\n",
    "dataset.get_shape()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Featurizing datapoint 0\n",
      "Featurizing datapoint 1000\n",
      "Featurizing datapoint 2000\n",
      "Featurizing datapoint 3000\n",
      "Featurizing datapoint 4000\n",
      "Featurizing datapoint 5000\n",
      "Featurizing datapoint 6000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "RDKit ERROR: [09:45:07] Explicit valence for atom # 1 Cl, 4, is greater than permitted\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "error in smile: O=[Cl]=O\n",
      "Featurizing datapoint 7000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "RDKit ERROR: [09:45:13] Explicit valence for atom # 3 B, 4, is greater than permitted\n",
      "RDKit ERROR: [09:45:13] Explicit valence for atom # 1 Cl, 9, is greater than permitted\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "error in smile: OB1O[B]2(O)OB(O)O[B](O)(O1)O2\n",
      "error in smile: O=[Cl-](=O)(=O)=O\n",
      "Featurizing datapoint 8000\n",
      "Featurizing datapoint 9000\n",
      "Featurizing datapoint 10000\n",
      "Featurizing datapoint 11000\n",
      "Featurizing datapoint 12000\n",
      "Featurizing datapoint 13000\n",
      "Featurizing datapoint 14000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "RDKit ERROR: [09:45:40] Explicit valence for atom # 0 P, 11, is greater than permitted\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "error in smile: [P](OCC=C(C)C)(OCC=C(C)C)(=O)(OP(OCC=C(C)C)(OCC=C(C)C)=O)(CC=C(C)C)(CC=C(C)C)(CC=C(C)C)(CC=C(C)C)(CC=C(C)C)CC=C(C)C\n",
      "Featurizing datapoint 15000\n",
      "Featurizing datapoint 16000\n",
      "Featurizing datapoint 17000\n",
      "Featurizing datapoint 18000\n",
      "Featurizing datapoint 19000\n",
      "Featurizing datapoint 20000\n",
      "Featurizing datapoint 21000\n",
      "Featurizing datapoint 22000\n",
      "Featurizing datapoint 23000\n",
      "Elements with indexes:  [6257, 7708, 7709, 14244]  were removed due to the presence of NAs!\n",
      "The elements in question are:  ['O=[Cl]=O' 'OB1O[B]2(O)OB(O)O[B](O)(O1)O2' 'O=[Cl-](=O)(=O)=O'\n",
      " '[P](OCC=C(C)C)(OCC=C(C)C)(=O)(OP(OCC=C(C)C)(OCC=C(C)C)=O)(CC=C(C)C)(CC=C(C)C)(CC=C(C)C)(CC=C(C)C)(CC=C(C)C)CC=C(C)C']\n",
      "Mols_shape:  23286\n",
      "Features_shape:  (23286, 2048)\n",
      "Labels_shape:  (23286,)\n"
     ]
    }
   ],
   "source": [
    "#Featurization\n",
    "dataset = MorganFingerprint().featurize(dataset)\n",
    "dataset.get_shape()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mols_shape:  23286\n",
      "Features_shape:  (23286, 44)\n",
      "Labels_shape:  (23286,)\n"
     ]
    }
   ],
   "source": [
    "#Feature Selection\n",
    "dataset = LowVarianceFS(0.15).featureSelection(dataset)\n",
    "dataset.get_shape()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13968, 44)\n",
      "(4656, 44)\n",
      "(4656, 44)\n"
     ]
    }
   ],
   "source": [
    "#Data Split\n",
    "splitter = SingletaskStratifiedSplitter()\n",
    "train_dataset, valid_dataset, test_dataset = splitter.train_valid_test_split(dataset=dataset, frac_train=0.6, \n",
    "                                                                             frac_valid=0.2, frac_test=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "#Scikit-Learn Random Forest\n",
    "rf = RandomForestClassifier()\n",
    "model = SklearnModel(model=rf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing Stratified K-fold split\n",
      "(15524, 44)\n",
      "(7762, 44)\n",
      "(15524, 44)\n",
      "(7762, 44)\n",
      "(15524, 44)\n",
      "(7762, 44)\n",
      "\n",
      "Split 1 :\n",
      "Train Score: \n",
      "roc_auc_score: \n",
      " 0.8163949821607831\n",
      "Test Score: \n",
      "roc_auc_score: \n",
      " 0.6438296168458691\n",
      "\n",
      "Split 2 :\n",
      "Train Score: \n",
      "roc_auc_score: \n",
      " 0.8150548316679306\n",
      "Test Score: \n",
      "roc_auc_score: \n",
      " 0.6375165282613524\n",
      "\n",
      "Split 3 :\n",
      "Train Score: \n",
      "roc_auc_score: \n",
      " 0.7997442808924287\n",
      "Test Score: \n",
      "roc_auc_score: \n",
      " 0.652829387457756\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(SklearnModel(model=RandomForestClassifier()),\n",
       " 0.7997442808924287,\n",
       " 0.652829387457756,\n",
       " [0.8163949821607831, 0.8150548316679306, 0.7997442808924287],\n",
       " [0.6438296168458691, 0.6375165282613524, 0.652829387457756],\n",
       " 0.8103980315737142,\n",
       " 0.6447251775216591)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#cross validation\n",
    "model.cross_validate(dataset, Metric(roc_auc_score), folds=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier()"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# model training\n",
    "model.fit(train_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Dataset: \n",
      "roc_auc_score: \n",
      " 0.8043855865293515\n",
      "precision_score: \n",
      " 0.8977695167286245\n",
      "accuracy_score: \n",
      " 0.9742268041237113\n",
      "confusion_matrix: \n",
      " [[13125    55]\n",
      " [  305   483]]\n",
      "classification_report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99     13180\n",
      "           1       0.90      0.61      0.73       788\n",
      "\n",
      "    accuracy                           0.97     13968\n",
      "   macro avg       0.94      0.80      0.86     13968\n",
      "weighted avg       0.97      0.97      0.97     13968\n",
      "\n",
      "WARNING: task averager  cannot perform reduce with flexible type\n",
      "\n",
      "\n",
      "Validation Dataset: \n",
      "roc_auc_score: \n",
      " 0.6419937075674781\n",
      "precision_score: \n",
      " 0.541095890410959\n",
      "accuracy_score: \n",
      " 0.9458762886597938\n",
      "confusion_matrix: \n",
      " [[4325   67]\n",
      " [ 185   79]]\n",
      "classification_report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.98      0.97      4392\n",
      "           1       0.54      0.30      0.39       264\n",
      "\n",
      "    accuracy                           0.95      4656\n",
      "   macro avg       0.75      0.64      0.68      4656\n",
      "weighted avg       0.94      0.95      0.94      4656\n",
      "\n",
      "WARNING: task averager  cannot perform reduce with flexible type\n",
      "\n",
      "\n",
      "Test Dataset: \n",
      "roc_auc_score: \n",
      " 0.6346456366948171\n",
      "precision_score: \n",
      " 0.5357142857142857\n",
      "accuracy_score: \n",
      " 0.945446735395189\n",
      "confusion_matrix: \n",
      " [[4327   65]\n",
      " [ 189   75]]\n",
      "classification_report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.99      0.97      4392\n",
      "           1       0.54      0.28      0.37       264\n",
      "\n",
      "    accuracy                           0.95      4656\n",
      "   macro avg       0.75      0.63      0.67      4656\n",
      "weighted avg       0.93      0.95      0.94      4656\n",
      "\n",
      "WARNING: task averager  cannot perform reduce with flexible type\n"
     ]
    }
   ],
   "source": [
    "metrics = [Metric(roc_auc_score), Metric(precision_score), Metric(accuracy_score), Metric(confusion_matrix), \n",
    "           Metric(classification_report)]\n",
    "\n",
    "# evaluate the model\n",
    "print('Training Dataset: ')\n",
    "train_score = model.evaluate(train_dataset, metrics)\n",
    "\n",
    "print('\\n\\nValidation Dataset: ')\n",
    "valid_score = model.evaluate(valid_dataset, metrics)\n",
    "\n",
    "print('\\n\\nTest Dataset: ')\n",
    "test_score = model.evaluate(test_dataset, metrics)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "#Build a model function for hyperparameter optimization\n",
    "def rf_model_builder(n_estimators=10, max_features='auto', class_weight={0: 1., 1: 1.}):\n",
    "    rf_model = RandomForestClassifier(n_estimators=n_estimators, max_features=max_features, \n",
    "                                      class_weight=class_weight)\n",
    "    return rf_model\n",
    "\n",
    "params_dict_rf = {\"n_estimators\": [10, 100],\n",
    "                  \"max_features\": [\"auto\", \"sqrt\", \"log2\", None],\n",
    "                  \"class_weight\": [{0: 1., 1: 1.}, {0: 1., 1: 5}, {0: 1., 1: 10}]\n",
    "                  }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MODE:  classification\n",
      "Fitting 15 random models from a space of 24 possible models.\n",
      "Fitting model 1/15\n",
      "hyperparameters: {'n_estimators': 10, 'max_features': 'sqrt', 'class_weight': {0: 1.0, 1: 1.0}}\n",
      "expected str, bytes or os.PathLike object, not NoneType\n",
      "accuracy_score: \n",
      " 0.9450171821305842\n",
      "Model 1/15, Metric accuracy_score, Validation set 1: 0.945017\n",
      "\tbest_validation_score so far: 0.945017\n",
      "Fitting model 2/15\n",
      "hyperparameters: {'n_estimators': 10, 'max_features': 'sqrt', 'class_weight': {0: 1.0, 1: 5}}\n",
      "expected str, bytes or os.PathLike object, not NoneType\n",
      "accuracy_score: \n",
      " 0.9372852233676976\n",
      "Model 2/15, Metric accuracy_score, Validation set 2: 0.937285\n",
      "\tbest_validation_score so far: 0.945017\n",
      "Fitting model 3/15\n",
      "hyperparameters: {'n_estimators': 10, 'max_features': 'sqrt', 'class_weight': {0: 1.0, 1: 10}}\n",
      "expected str, bytes or os.PathLike object, not NoneType\n",
      "accuracy_score: \n",
      " 0.9317010309278351\n",
      "Model 3/15, Metric accuracy_score, Validation set 3: 0.931701\n",
      "\tbest_validation_score so far: 0.945017\n",
      "Fitting model 4/15\n",
      "hyperparameters: {'n_estimators': 10, 'max_features': 'log2', 'class_weight': {0: 1.0, 1: 5}}\n",
      "expected str, bytes or os.PathLike object, not NoneType\n",
      "accuracy_score: \n",
      " 0.9370704467353952\n",
      "Model 4/15, Metric accuracy_score, Validation set 4: 0.937070\n",
      "\tbest_validation_score so far: 0.945017\n",
      "Fitting model 5/15\n",
      "hyperparameters: {'n_estimators': 10, 'max_features': 'log2', 'class_weight': {0: 1.0, 1: 10}}\n",
      "expected str, bytes or os.PathLike object, not NoneType\n",
      "accuracy_score: \n",
      " 0.9297680412371134\n",
      "Model 5/15, Metric accuracy_score, Validation set 5: 0.929768\n",
      "\tbest_validation_score so far: 0.945017\n",
      "Fitting model 6/15\n",
      "hyperparameters: {'n_estimators': 10, 'max_features': None, 'class_weight': {0: 1.0, 1: 1.0}}\n",
      "expected str, bytes or os.PathLike object, not NoneType\n",
      "accuracy_score: \n",
      " 0.9428694158075601\n",
      "Model 6/15, Metric accuracy_score, Validation set 6: 0.942869\n",
      "\tbest_validation_score so far: 0.945017\n",
      "Fitting model 7/15\n",
      "hyperparameters: {'n_estimators': 100, 'max_features': 'auto', 'class_weight': {0: 1.0, 1: 1.0}}\n",
      "expected str, bytes or os.PathLike object, not NoneType\n",
      "accuracy_score: \n",
      " 0.948668384879725\n",
      "Model 7/15, Metric accuracy_score, Validation set 7: 0.948668\n",
      "\tbest_validation_score so far: 0.948668\n",
      "Fitting model 8/15\n",
      "hyperparameters: {'n_estimators': 100, 'max_features': 'auto', 'class_weight': {0: 1.0, 1: 5}}\n",
      "expected str, bytes or os.PathLike object, not NoneType\n",
      "accuracy_score: \n",
      " 0.9405068728522337\n",
      "Model 8/15, Metric accuracy_score, Validation set 8: 0.940507\n",
      "\tbest_validation_score so far: 0.948668\n",
      "Fitting model 9/15\n",
      "hyperparameters: {'n_estimators': 100, 'max_features': 'auto', 'class_weight': {0: 1.0, 1: 10}}\n",
      "expected str, bytes or os.PathLike object, not NoneType\n",
      "accuracy_score: \n",
      " 0.9349226804123711\n",
      "Model 9/15, Metric accuracy_score, Validation set 9: 0.934923\n",
      "\tbest_validation_score so far: 0.948668\n",
      "Fitting model 10/15\n",
      "hyperparameters: {'n_estimators': 100, 'max_features': 'sqrt', 'class_weight': {0: 1.0, 1: 1.0}}\n",
      "expected str, bytes or os.PathLike object, not NoneType\n",
      "accuracy_score: \n",
      " 0.9503865979381443\n",
      "Model 10/15, Metric accuracy_score, Validation set 10: 0.950387\n",
      "\tbest_validation_score so far: 0.950387\n",
      "Fitting model 11/15\n",
      "hyperparameters: {'n_estimators': 100, 'max_features': 'sqrt', 'class_weight': {0: 1.0, 1: 10}}\n",
      "expected str, bytes or os.PathLike object, not NoneType\n",
      "accuracy_score: \n",
      " 0.9349226804123711\n",
      "Model 11/15, Metric accuracy_score, Validation set 11: 0.934923\n",
      "\tbest_validation_score so far: 0.950387\n",
      "Fitting model 12/15\n",
      "hyperparameters: {'n_estimators': 100, 'max_features': 'log2', 'class_weight': {0: 1.0, 1: 1.0}}\n",
      "expected str, bytes or os.PathLike object, not NoneType\n",
      "accuracy_score: \n",
      " 0.9497422680412371\n",
      "Model 12/15, Metric accuracy_score, Validation set 12: 0.949742\n",
      "\tbest_validation_score so far: 0.950387\n",
      "Fitting model 13/15\n",
      "hyperparameters: {'n_estimators': 100, 'max_features': 'log2', 'class_weight': {0: 1.0, 1: 5}}\n",
      "expected str, bytes or os.PathLike object, not NoneType\n",
      "accuracy_score: \n",
      " 0.9450171821305842\n",
      "Model 13/15, Metric accuracy_score, Validation set 13: 0.945017\n",
      "\tbest_validation_score so far: 0.950387\n",
      "Fitting model 14/15\n",
      "hyperparameters: {'n_estimators': 100, 'max_features': None, 'class_weight': {0: 1.0, 1: 5}}\n",
      "expected str, bytes or os.PathLike object, not NoneType\n",
      "accuracy_score: \n",
      " 0.9430841924398625\n",
      "Model 14/15, Metric accuracy_score, Validation set 14: 0.943084\n",
      "\tbest_validation_score so far: 0.950387\n",
      "Fitting model 15/15\n",
      "hyperparameters: {'n_estimators': 100, 'max_features': None, 'class_weight': {0: 1.0, 1: 10}}\n",
      "expected str, bytes or os.PathLike object, not NoneType\n",
      "accuracy_score: \n",
      " 0.9334192439862543\n",
      "Model 15/15, Metric accuracy_score, Validation set 15: 0.933419\n",
      "\tbest_validation_score so far: 0.950387\n",
      "accuracy_score: \n",
      " 0.9802405498281787\n",
      "Best hyperparameters: (100, 'sqrt', {0: 1.0, 1: 1.0})\n",
      "train_score: 0.980241\n",
      "validation_score: 0.950387\n",
      "#################\n",
      "(100, 'sqrt', {0: 1.0, 1: 1.0})\n",
      "SklearnModel(mode='/tmp/tmp_isfmccm',\n",
      "             model=RandomForestClassifier(class_weight={0: 1.0, 1: 1.0},\n",
      "                                          max_features='sqrt'))\n",
      "roc_auc_score: \n",
      " 0.6477324364115397\n",
      "precision_score: \n",
      " 0.5955882352941176\n",
      "accuracy_score: \n",
      " 0.9490979381443299\n",
      "confusion_matrix: \n",
      " [[4338   55]\n",
      " [ 182   81]]\n",
      "classification_report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.99      0.97      4393\n",
      "           1       0.60      0.31      0.41       263\n",
      "\n",
      "    accuracy                           0.95      4656\n",
      "   macro avg       0.78      0.65      0.69      4656\n",
      "weighted avg       0.94      0.95      0.94      4656\n",
      "\n",
      "WARNING: task averager  cannot perform reduce with flexible type\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'roc_auc_score': 0.6477324364115397,\n",
       " 'precision_score': 0.5955882352941176,\n",
       " 'accuracy_score': 0.9490979381443299,\n",
       " 'confusion_matrix': 1164.0,\n",
       " 'classification_report': None}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Hyperparameter Optimization\n",
    "optimizer = HyperparamOpt_Valid(rf_model_builder)\n",
    "\n",
    "best_rf, best_hyperparams, all_results = optimizer.hyperparam_search(params_dict_rf, \n",
    "                                                                     train_dataset, \n",
    "                                                                     valid_dataset, \n",
    "                                                                     Metric(accuracy_score))\n",
    "\n",
    "print('#################')\n",
    "print(best_hyperparams)\n",
    "print(best_rf)\n",
    "\n",
    "#Evaluate model\n",
    "best_rf.evaluate(test_dataset, metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MODEL TYPE:  sklearn\n",
      "Fitting 10 random models from a space of 24 possible models.\n",
      "RandomForestClassifier(class_weight={0: 1.0, 1: 1.0}, n_estimators=10)\n",
      "\n",
      " \n",
      " Best accuracy: 0.936068 using {'n_estimators': 10, 'max_features': 'auto', 'class_weight': {0: 1.0, 1: 1.0}}\n",
      "\n",
      " accuracy: 0.921893 (0.005206) with: {'n_estimators': 100, 'max_features': 'sqrt', 'class_weight': {0: 1.0, 1: 10}} \n",
      "\n",
      "\n",
      " accuracy: 0.926904 (0.005209) with: {'n_estimators': 100, 'max_features': 'log2', 'class_weight': {0: 1.0, 1: 5}} \n",
      "\n",
      "\n",
      " accuracy: 0.924399 (0.005333) with: {'n_estimators': 10, 'max_features': 'log2', 'class_weight': {0: 1.0, 1: 5}} \n",
      "\n",
      "\n",
      " accuracy: 0.925687 (0.004650) with: {'n_estimators': 10, 'max_features': 'auto', 'class_weight': {0: 1.0, 1: 5}} \n",
      "\n",
      "\n",
      " accuracy: 0.924255 (0.005035) with: {'n_estimators': 10, 'max_features': None, 'class_weight': {0: 1.0, 1: 5}} \n",
      "\n",
      "\n",
      " accuracy: 0.927549 (0.005090) with: {'n_estimators': 100, 'max_features': 'sqrt', 'class_weight': {0: 1.0, 1: 5}} \n",
      "\n",
      "\n",
      " accuracy: 0.922322 (0.005010) with: {'n_estimators': 100, 'max_features': 'log2', 'class_weight': {0: 1.0, 1: 10}} \n",
      "\n",
      "\n",
      " accuracy: 0.936068 (0.002603) with: {'n_estimators': 10, 'max_features': 'auto', 'class_weight': {0: 1.0, 1: 1.0}} \n",
      "\n",
      "\n",
      " accuracy: 0.924112 (0.006201) with: {'n_estimators': 100, 'max_features': None, 'class_weight': {0: 1.0, 1: 5}} \n",
      "\n",
      "\n",
      " accuracy: 0.927907 (0.003942) with: {'n_estimators': 100, 'max_features': 'auto', 'class_weight': {0: 1.0, 1: 5}} \n",
      "\n",
      "RandomForestClassifier(class_weight={0: 1.0, 1: 1.0}, n_estimators=10)\n",
      "#################\n",
      "{'n_estimators': 10, 'max_features': 'auto', 'class_weight': {0: 1.0, 1: 1.0}}\n",
      "SklearnModel(mode='classification',\n",
      "             model=RandomForestClassifier(class_weight={0: 1.0, 1: 1.0},\n",
      "                                          n_estimators=10))\n"
     ]
    }
   ],
   "source": [
    "#Hyperparameter Optimization with CV\n",
    "optimizer = HyperparamOpt_CV(rf_model_builder)\n",
    "\n",
    "best_rf, best_hyperparams, all_results = optimizer.hyperparam_search('sklearn',\n",
    "                                                                     params_dict_rf, \n",
    "                                                                     train_dataset,  \n",
    "                                                                     'accuracy', \n",
    "                                                                     cv=3,\n",
    "                                                                     n_iter_search=10)\n",
    "\n",
    "print('#################')\n",
    "print(best_hyperparams)\n",
    "print(best_rf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "roc_auc_score: \n",
      " 0.6562938445972205\n",
      "precision_score: \n",
      " 0.5209580838323353\n",
      "accuracy_score: \n",
      " 0.9450171821305842\n",
      "confusion_matrix: \n",
      " [[4313   80]\n",
      " [ 176   87]]\n",
      "classification_report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.98      0.97      4393\n",
      "           1       0.52      0.33      0.40       263\n",
      "\n",
      "    accuracy                           0.95      4656\n",
      "   macro avg       0.74      0.66      0.69      4656\n",
      "weighted avg       0.94      0.95      0.94      4656\n",
      "\n",
      "WARNING: task averager  cannot perform reduce with flexible type\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'roc_auc_score': 0.6562938445972205,\n",
       " 'precision_score': 0.5209580838323353,\n",
       " 'accuracy_score': 0.9450171821305842,\n",
       " 'confusion_matrix': 1164.0,\n",
       " 'classification_report': None}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Evaluate model\n",
    "best_rf.evaluate(test_dataset, metrics)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}