video_hyperparam.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_classification\n",
    "\n",
    "X, y = make_classification(n_samples=5000, n_features=10, n_redundant=2, n_informative=5, n_clusters_per_class=2, flip_y=0.09, class_sep = 1.1,\n",
    "                          random_state=4184)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5000, 10)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state = 42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4000, 10)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4000,)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "clf = DecisionTreeClassifier(random_state=42, criterion=\"entropy\", min_samples_split=10, max_depth=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(class_weight=None, criterion='entropy', max_depth=5,\n",
       "                       max_features=None, max_leaf_nodes=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=10,\n",
       "                       min_weight_fraction_leaf=0.0, presort=False,\n",
       "                       random_state=42, splitter='best')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(class_weight=None, criterion='entropy', max_depth=5,\n",
       "                       max_features=None, max_leaf_nodes=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=10,\n",
       "                       min_weight_fraction_leaf=0.0, presort=False,\n",
       "                       random_state=42, splitter='best')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[432  63]\n",
      " [189 316]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix, f1_score\n",
    "\n",
    "y_pred = clf.predict(X_test)\n",
    "print(confusion_matrix(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 score = 0.71\n"
     ]
    }
   ],
   "source": [
    "print(\"F1 score = {:.2f}\".format(f1_score(y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 32.7 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "             estimator=DecisionTreeClassifier(class_weight=None,\n",
       "                                              criterion='gini', max_depth=None,\n",
       "                                              max_features=None,\n",
       "                                              max_leaf_nodes=None,\n",
       "                                              min_impurity_decrease=0.0,\n",
       "                                              min_impurity_split=None,\n",
       "                                              min_samples_leaf=1,\n",
       "                                              min_samples_split=2,\n",
       "                                              min_weight_fraction_leaf=0.0,\n",
       "                                              presort=False, random_state=42,\n",
       "                                              splitter='best'),\n",
       "             iid='warn', n_jobs=3,\n",
       "             param_grid={'criterion': ('gini', 'entropy'),\n",
       "                         'max_depth': [2, 10, 50, 100, 200],\n",
       "                         'max_leaf_nodes': [None, 5, 10, 50, 100],\n",
       "                         'min_impurity_decrease': [0, 0.1, 0.2],\n",
       "                         'min_samples_leaf': [1, 10, 50],\n",
       "                         'min_samples_split': [2, 10, 50]},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "             scoring='f1', verbose=0)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "base_clf = DecisionTreeClassifier(splitter=\"best\", class_weight=None, random_state=42)\n",
    "parameters_grid = {'criterion': ('gini', 'entropy'),\n",
    "                  'max_depth': [2, 10, 50, 100, 200],\n",
    "                  'max_leaf_nodes': [None, 5, 10, 50, 100],\n",
    "                  'min_impurity_decrease': [0, 0.1, 0.2],\n",
    "                  'min_samples_leaf': [1, 10, 50],\n",
    "                  'min_samples_split': [2, 10, 50]}\n",
    "clf_gs = GridSearchCV(base_clf, param_grid=parameters_grid, scoring=\"f1\", n_jobs=3, cv=5, return_train_score=True)\n",
    "%time clf_gs.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'criterion': 'entropy',\n",
       " 'max_depth': 50,\n",
       " 'max_leaf_nodes': 100,\n",
       " 'min_impurity_decrease': 0,\n",
       " 'min_samples_leaf': 10,\n",
       " 'min_samples_split': 2}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf_gs.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[405  90]\n",
      " [111 394]]\n",
      "F1 Score = 0.80\n"
     ]
    }
   ],
   "source": [
    "y_pred_gs = clf_gs.predict(X_test)\n",
    "print(confusion_matrix(y_test, y_pred_gs))\n",
    "print(\"F1 Score = {:.2f}\".format(f1_score(y_test, y_pred_gs)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cv_results_to_df(cv_results):\n",
    "    results = pd.DataFrame(list(cv_results['params']))\n",
    "    results['mean_fit_time'] = cv_results['mean_fit_time']\n",
    "    results['mean_score_time'] = cv_results['mean_score_time']\n",
    "    results['mean_train_score'] = cv_results['mean_train_score']\n",
    "    results['std_train_score'] = cv_results['std_train_score']\n",
    "    results['mean_test_score'] = cv_results['mean_test_score']\n",
    "    results['std_test_score'] = cv_results['std_test_score']\n",
    "    results['rank_test_score'] = cv_results['rank_test_score']\n",
    "\n",
    "    results = results.sort_values(['mean_test_score'], ascending=False)\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>criterion</th>\n",
       "      <th>max_depth</th>\n",
       "      <th>max_leaf_nodes</th>\n",
       "      <th>min_impurity_decrease</th>\n",
       "      <th>min_samples_leaf</th>\n",
       "      <th>min_samples_split</th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>mean_train_score</th>\n",
       "      <th>std_train_score</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1326</th>\n",
       "      <td>entropy</td>\n",
       "      <td>200</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0.038381</td>\n",
       "      <td>0.001000</td>\n",
       "      <td>0.879076</td>\n",
       "      <td>0.004364</td>\n",
       "      <td>0.815684</td>\n",
       "      <td>0.008439</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1327</th>\n",
       "      <td>entropy</td>\n",
       "      <td>200</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>0.042800</td>\n",
       "      <td>0.001292</td>\n",
       "      <td>0.879076</td>\n",
       "      <td>0.004364</td>\n",
       "      <td>0.815684</td>\n",
       "      <td>0.008439</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1057</th>\n",
       "      <td>entropy</td>\n",
       "      <td>50</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>0.040398</td>\n",
       "      <td>0.001000</td>\n",
       "      <td>0.879076</td>\n",
       "      <td>0.004364</td>\n",
       "      <td>0.815684</td>\n",
       "      <td>0.008439</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1056</th>\n",
       "      <td>entropy</td>\n",
       "      <td>50</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0.040500</td>\n",
       "      <td>0.001200</td>\n",
       "      <td>0.879076</td>\n",
       "      <td>0.004364</td>\n",
       "      <td>0.815684</td>\n",
       "      <td>0.008439</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1191</th>\n",
       "      <td>entropy</td>\n",
       "      <td>100</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0.042600</td>\n",
       "      <td>0.001201</td>\n",
       "      <td>0.879076</td>\n",
       "      <td>0.004364</td>\n",
       "      <td>0.815684</td>\n",
       "      <td>0.008439</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>837</th>\n",
       "      <td>entropy</td>\n",
       "      <td>10</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.020499</td>\n",
       "      <td>0.001200</td>\n",
       "      <td>0.678763</td>\n",
       "      <td>0.008997</td>\n",
       "      <td>0.667952</td>\n",
       "      <td>0.029160</td>\n",
       "      <td>1315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>838</th>\n",
       "      <td>entropy</td>\n",
       "      <td>10</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>0.020998</td>\n",
       "      <td>0.001202</td>\n",
       "      <td>0.678763</td>\n",
       "      <td>0.008997</td>\n",
       "      <td>0.667952</td>\n",
       "      <td>0.029160</td>\n",
       "      <td>1315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>839</th>\n",
       "      <td>entropy</td>\n",
       "      <td>10</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>50</td>\n",
       "      <td>0.021900</td>\n",
       "      <td>0.001000</td>\n",
       "      <td>0.678763</td>\n",
       "      <td>0.008997</td>\n",
       "      <td>0.667952</td>\n",
       "      <td>0.029160</td>\n",
       "      <td>1315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>840</th>\n",
       "      <td>entropy</td>\n",
       "      <td>10</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0.019999</td>\n",
       "      <td>0.001101</td>\n",
       "      <td>0.678763</td>\n",
       "      <td>0.008997</td>\n",
       "      <td>0.667952</td>\n",
       "      <td>0.029160</td>\n",
       "      <td>1315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1109</th>\n",
       "      <td>entropy</td>\n",
       "      <td>100</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>50</td>\n",
       "      <td>0.018993</td>\n",
       "      <td>0.001014</td>\n",
       "      <td>0.678763</td>\n",
       "      <td>0.008997</td>\n",
       "      <td>0.667952</td>\n",
       "      <td>0.029160</td>\n",
       "      <td>1315</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1350 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     criterion  max_depth  max_leaf_nodes  min_impurity_decrease  \\\n",
       "1326   entropy        200           100.0                    0.0   \n",
       "1327   entropy        200           100.0                    0.0   \n",
       "1057   entropy         50           100.0                    0.0   \n",
       "1056   entropy         50           100.0                    0.0   \n",
       "1191   entropy        100           100.0                    0.0   \n",
       "...        ...        ...             ...                    ...   \n",
       "837    entropy         10             5.0                    0.0   \n",
       "838    entropy         10             5.0                    0.0   \n",
       "839    entropy         10             5.0                    0.0   \n",
       "840    entropy         10             5.0                    0.0   \n",
       "1109   entropy        100             5.0                    0.0   \n",
       "\n",
       "      min_samples_leaf  min_samples_split  mean_fit_time  mean_score_time  \\\n",
       "1326                10                  2       0.038381         0.001000   \n",
       "1327                10                 10       0.042800         0.001292   \n",
       "1057                10                 10       0.040398         0.001000   \n",
       "1056                10                  2       0.040500         0.001200   \n",
       "1191                10                  2       0.042600         0.001201   \n",
       "...                ...                ...            ...              ...   \n",
       "837                  1                  2       0.020499         0.001200   \n",
       "838                  1                 10       0.020998         0.001202   \n",
       "839                  1                 50       0.021900         0.001000   \n",
       "840                 10                  2       0.019999         0.001101   \n",
       "1109                 1                 50       0.018993         0.001014   \n",
       "\n",
       "      mean_train_score  std_train_score  mean_test_score  std_test_score  \\\n",
       "1326          0.879076         0.004364         0.815684        0.008439   \n",
       "1327          0.879076         0.004364         0.815684        0.008439   \n",
       "1057          0.879076         0.004364         0.815684        0.008439   \n",
       "1056          0.879076         0.004364         0.815684        0.008439   \n",
       "1191          0.879076         0.004364         0.815684        0.008439   \n",
       "...                ...              ...              ...             ...   \n",
       "837           0.678763         0.008997         0.667952        0.029160   \n",
       "838           0.678763         0.008997         0.667952        0.029160   \n",
       "839           0.678763         0.008997         0.667952        0.029160   \n",
       "840           0.678763         0.008997         0.667952        0.029160   \n",
       "1109          0.678763         0.008997         0.667952        0.029160   \n",
       "\n",
       "      rank_test_score  \n",
       "1326                1  \n",
       "1327                1  \n",
       "1057                1  \n",
       "1056                1  \n",
       "1191                1  \n",
       "...               ...  \n",
       "837              1315  \n",
       "838              1315  \n",
       "839              1315  \n",
       "840              1315  \n",
       "1109             1315  \n",
       "\n",
       "[1350 rows x 13 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_gs = cv_results_to_df(clf_gs.cv_results_)\n",
    "results_gs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 1min\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=5, error_score='raise-deprecating',\n",
       "                   estimator=DecisionTreeClassifier(class_weight=None,\n",
       "                                                    criterion='gini',\n",
       "                                                    max_depth=None,\n",
       "                                                    max_features=None,\n",
       "                                                    max_leaf_nodes=None,\n",
       "                                                    min_impurity_decrease=0.0,\n",
       "                                                    min_impurity_split=None,\n",
       "                                                    min_samples_leaf=1,\n",
       "                                                    min_samples_split=2,\n",
       "                                                    min_weight_fraction_leaf=0.0,\n",
       "                                                    presort=False,\n",
       "                                                    random_state=42,\n",
       "                                                    splitter='best'),\n",
       "                   iid='war...\n",
       "                                        'min_impurity_decrease': <scipy.stats._distn_infrastructure.rv_frozen object at 0x00000121B2DE4898>,\n",
       "                                        'min_samples_leaf': <scipy.stats._distn_infrastructure.rv_frozen object at 0x00000121B2F02CF8>,\n",
       "                                        'min_samples_split': <scipy.stats._distn_infrastructure.rv_frozen object at 0x00000121B2DE4B38>},\n",
       "                   pre_dispatch='2*n_jobs', random_state=None, refit=True,\n",
       "                   return_train_score=True, scoring='f1', verbose=0)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from scipy.stats import randint, uniform\n",
    "\n",
    "parameters_dist = {\n",
    "              'criterion': [\"entropy\", \"gini\", \"gini\"],\n",
    "              'max_depth': randint(2, 200),\n",
    "              'max_features': [None, \"auto\"],\n",
    "              'max_leaf_nodes': randint(5, 500),\n",
    "              'min_impurity_decrease': uniform(0.0, 0.5),\n",
    "              'min_samples_leaf': randint(2, 50),\n",
    "              'min_samples_split': randint(2, 50)}\n",
    "\n",
    "clf_rs = RandomizedSearchCV(base_clf, param_distributions=parameters_dist, cv=5, n_jobs=3, scoring=\"f1\", return_train_score=True, n_iter=5000)\n",
    "\n",
    "%time clf_rs.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[410  85]\n",
      " [ 93 412]]\n",
      "F1 Score = 0.82\n"
     ]
    }
   ],
   "source": [
    "y_pred_rs = clf_rs.predict(X_test)\n",
    "print(confusion_matrix(y_test, y_pred_rs))\n",
    "print(\"F1 Score = {:.2f}\".format(f1_score(y_test, y_pred_rs)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>criterion</th>\n",
       "      <th>max_depth</th>\n",
       "      <th>max_features</th>\n",
       "      <th>max_leaf_nodes</th>\n",
       "      <th>min_impurity_decrease</th>\n",
       "      <th>min_samples_leaf</th>\n",
       "      <th>min_samples_split</th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>mean_train_score</th>\n",
       "      <th>std_train_score</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3073</th>\n",
       "      <td>gini</td>\n",
       "      <td>75</td>\n",
       "      <td>None</td>\n",
       "      <td>75</td>\n",
       "      <td>0.001162</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0.028106</td>\n",
       "      <td>0.001501</td>\n",
       "      <td>0.864988</td>\n",
       "      <td>0.007504</td>\n",
       "      <td>0.801445</td>\n",
       "      <td>0.013550</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1641</th>\n",
       "      <td>entropy</td>\n",
       "      <td>113</td>\n",
       "      <td>None</td>\n",
       "      <td>373</td>\n",
       "      <td>0.001492</td>\n",
       "      <td>20</td>\n",
       "      <td>34</td>\n",
       "      <td>0.037483</td>\n",
       "      <td>0.001116</td>\n",
       "      <td>0.850497</td>\n",
       "      <td>0.003251</td>\n",
       "      <td>0.795571</td>\n",
       "      <td>0.008642</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1684</th>\n",
       "      <td>entropy</td>\n",
       "      <td>31</td>\n",
       "      <td>None</td>\n",
       "      <td>347</td>\n",
       "      <td>0.002441</td>\n",
       "      <td>27</td>\n",
       "      <td>24</td>\n",
       "      <td>0.028614</td>\n",
       "      <td>0.001007</td>\n",
       "      <td>0.828335</td>\n",
       "      <td>0.005931</td>\n",
       "      <td>0.792448</td>\n",
       "      <td>0.014432</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4266</th>\n",
       "      <td>gini</td>\n",
       "      <td>15</td>\n",
       "      <td>None</td>\n",
       "      <td>472</td>\n",
       "      <td>0.000939</td>\n",
       "      <td>32</td>\n",
       "      <td>15</td>\n",
       "      <td>0.019216</td>\n",
       "      <td>0.001100</td>\n",
       "      <td>0.828934</td>\n",
       "      <td>0.007141</td>\n",
       "      <td>0.792255</td>\n",
       "      <td>0.008882</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>738</th>\n",
       "      <td>gini</td>\n",
       "      <td>86</td>\n",
       "      <td>None</td>\n",
       "      <td>246</td>\n",
       "      <td>0.000309</td>\n",
       "      <td>35</td>\n",
       "      <td>41</td>\n",
       "      <td>0.019399</td>\n",
       "      <td>0.001194</td>\n",
       "      <td>0.824625</td>\n",
       "      <td>0.006918</td>\n",
       "      <td>0.787084</td>\n",
       "      <td>0.014087</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>958</th>\n",
       "      <td>entropy</td>\n",
       "      <td>181</td>\n",
       "      <td>None</td>\n",
       "      <td>451</td>\n",
       "      <td>0.081685</td>\n",
       "      <td>48</td>\n",
       "      <td>36</td>\n",
       "      <td>0.011509</td>\n",
       "      <td>0.001101</td>\n",
       "      <td>0.544182</td>\n",
       "      <td>0.034021</td>\n",
       "      <td>0.525858</td>\n",
       "      <td>0.025312</td>\n",
       "      <td>4980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1136</th>\n",
       "      <td>entropy</td>\n",
       "      <td>134</td>\n",
       "      <td>None</td>\n",
       "      <td>33</td>\n",
       "      <td>0.085046</td>\n",
       "      <td>46</td>\n",
       "      <td>32</td>\n",
       "      <td>0.011411</td>\n",
       "      <td>0.001089</td>\n",
       "      <td>0.544182</td>\n",
       "      <td>0.034021</td>\n",
       "      <td>0.525858</td>\n",
       "      <td>0.025312</td>\n",
       "      <td>4980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3065</th>\n",
       "      <td>entropy</td>\n",
       "      <td>62</td>\n",
       "      <td>None</td>\n",
       "      <td>177</td>\n",
       "      <td>0.076913</td>\n",
       "      <td>37</td>\n",
       "      <td>20</td>\n",
       "      <td>0.012608</td>\n",
       "      <td>0.001092</td>\n",
       "      <td>0.544182</td>\n",
       "      <td>0.034021</td>\n",
       "      <td>0.525858</td>\n",
       "      <td>0.025312</td>\n",
       "      <td>4980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4771</th>\n",
       "      <td>gini</td>\n",
       "      <td>2</td>\n",
       "      <td>auto</td>\n",
       "      <td>438</td>\n",
       "      <td>0.010581</td>\n",
       "      <td>41</td>\n",
       "      <td>28</td>\n",
       "      <td>0.002600</td>\n",
       "      <td>0.001004</td>\n",
       "      <td>0.528540</td>\n",
       "      <td>0.039431</td>\n",
       "      <td>0.515660</td>\n",
       "      <td>0.053271</td>\n",
       "      <td>4999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2679</th>\n",
       "      <td>entropy</td>\n",
       "      <td>2</td>\n",
       "      <td>auto</td>\n",
       "      <td>215</td>\n",
       "      <td>0.015374</td>\n",
       "      <td>5</td>\n",
       "      <td>41</td>\n",
       "      <td>0.003700</td>\n",
       "      <td>0.000900</td>\n",
       "      <td>0.517566</td>\n",
       "      <td>0.020288</td>\n",
       "      <td>0.504014</td>\n",
       "      <td>0.031444</td>\n",
       "      <td>5000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5000 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     criterion  max_depth max_features  max_leaf_nodes  min_impurity_decrease  \\\n",
       "3073      gini         75         None              75               0.001162   \n",
       "1641   entropy        113         None             373               0.001492   \n",
       "1684   entropy         31         None             347               0.002441   \n",
       "4266      gini         15         None             472               0.000939   \n",
       "738       gini         86         None             246               0.000309   \n",
       "...        ...        ...          ...             ...                    ...   \n",
       "958    entropy        181         None             451               0.081685   \n",
       "1136   entropy        134         None              33               0.085046   \n",
       "3065   entropy         62         None             177               0.076913   \n",
       "4771      gini          2         auto             438               0.010581   \n",
       "2679   entropy          2         auto             215               0.015374   \n",
       "\n",
       "      min_samples_leaf  min_samples_split  mean_fit_time  mean_score_time  \\\n",
       "3073                 2                  3       0.028106         0.001501   \n",
       "1641                20                 34       0.037483         0.001116   \n",
       "1684                27                 24       0.028614         0.001007   \n",
       "4266                32                 15       0.019216         0.001100   \n",
       "738                 35                 41       0.019399         0.001194   \n",
       "...                ...                ...            ...              ...   \n",
       "958                 48                 36       0.011509         0.001101   \n",
       "1136                46                 32       0.011411         0.001089   \n",
       "3065                37                 20       0.012608         0.001092   \n",
       "4771                41                 28       0.002600         0.001004   \n",
       "2679                 5                 41       0.003700         0.000900   \n",
       "\n",
       "      mean_train_score  std_train_score  mean_test_score  std_test_score  \\\n",
       "3073          0.864988         0.007504         0.801445        0.013550   \n",
       "1641          0.850497         0.003251         0.795571        0.008642   \n",
       "1684          0.828335         0.005931         0.792448        0.014432   \n",
       "4266          0.828934         0.007141         0.792255        0.008882   \n",
       "738           0.824625         0.006918         0.787084        0.014087   \n",
       "...                ...              ...              ...             ...   \n",
       "958           0.544182         0.034021         0.525858        0.025312   \n",
       "1136          0.544182         0.034021         0.525858        0.025312   \n",
       "3065          0.544182         0.034021         0.525858        0.025312   \n",
       "4771          0.528540         0.039431         0.515660        0.053271   \n",
       "2679          0.517566         0.020288         0.504014        0.031444   \n",
       "\n",
       "      rank_test_score  \n",
       "3073                1  \n",
       "1641                2  \n",
       "1684                3  \n",
       "4266                4  \n",
       "738                 5  \n",
       "...               ...  \n",
       "958              4980  \n",
       "1136             4980  \n",
       "3065             4980  \n",
       "4771             4999  \n",
       "2679             5000  \n",
       "\n",
       "[5000 rows x 14 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_rs = cv_results_to_df(clf_rs.cv_results_)\n",
    "results_rs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "small_sklearn_kernel",
   "language": "python",
   "name": "small_sklearn_kernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}