RF_GridSearch_Recall_Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# All necessary imports\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import plot_confusion_matrix\n",
    "from sklearn.metrics import plot_roc_curve\n",
    "import pickle\n",
    "from sklearn.metrics import fbeta_score, make_scorer\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import precision_score\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Load training data and split labels and features (data)\n",
    "data = pd.read_csv(\"./diabetes/training_data(no_pre-diabetes).csv\")\n",
    "labels = data[\"Diabetes_012\"]\n",
    "del data[\"Diabetes_012\"] # deletes the labels from the data dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Split data in training and validation set, with for each a set of data and a set of corresponding labels\n",
    "training_data, validation_data, training_labels, validation_labels = train_test_split(data, labels, test_size=0.2, random_state=33)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "\"\"\"Function to calculate and print all the necessary scores for a model\n",
    "Negative is the non-diabetes class and Positive the diabetes class\"\"\"\n",
    "def print_multiple_scores(validation_labels, prediction_model):\n",
    "    # Determine the individual values of the confusion matrix\n",
    "    tn, fp, fn, tp = confusion_matrix(validation_labels, prediction_model).ravel()\n",
    "\n",
    "    # Calculate each individual score\n",
    "    model_sens = tp / (tp + fn)\n",
    "    model_spec = tn / (tn + fp)\n",
    "    model_prec = tp / (tp + fp)\n",
    "    model_fpr = fp / (tn + fp)\n",
    "    model_npv = tn / (fn + tn)\n",
    "\n",
    "    # Print all the scores for the user to see\n",
    "    print(\"Sensitity (PPV): \\t\", model_sens)\n",
    "    print(\"Specificity (TNR):  \", model_spec)\n",
    "    print(\"Precision: \\t\\t\\t\", model_prec)\n",
    "    print(\"False Positive Rate:\", model_fpr)\n",
    "    print(\"NPV: \\t\\t\\t\\t\", model_npv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Max features\n",
    "max_features = [\"auto\", 2, 4, 6, 8]\n",
    "\n",
    "# criterion\n",
    "criterion = [\"gini\", \"entropy\"]\n",
    "\n",
    "# Number of trees in the random forest\n",
    "n_estimators = [50, 100, 150, 200, 250, 500]\n",
    "\n",
    "# Create random_grid\n",
    "param_grid = {'max_features': max_features,\n",
    "              'criterion': criterion,\n",
    "              'n_estimators': n_estimators, }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "recall_scorer = make_scorer(recall_score,  pos_label=\"2.0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "cv_score_model = GridSearchCV(estimator=RandomForestClassifier(), param_grid=param_grid, scoring=\"recall_macro\", cv=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "GridSearchCV(cv=5, estimator=RandomForestClassifier(),\n             param_grid={'criterion': ['gini', 'entropy'],\n                         'max_features': ['auto', 2, 4, 6, 8],\n                         'n_estimators': [50, 100, 150, 200, 250, 500]},\n             scoring='recall_macro')"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_score_model.fit(training_data, training_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters:  {'criterion': 'gini', 'max_features': 8, 'n_estimators': 500}\n",
      "Accuracy GridSearchCV model:  0.8606358433464177\n"
     ]
    }
   ],
   "source": [
    "# Print out the best parameters according to GridSearchCV\n",
    "print(\"Best parameters: \", cv_score_model.best_params_)\n",
    "\n",
    "# Determine accuracy of model based on validation set\n",
    "pred_cv_score_model = cv_score_model.predict(validation_data)\n",
    "print(\"Accuracy GridSearchCV model: \", accuracy_score(validation_labels, pred_cv_score_model))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.9/dist-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function plot_confusion_matrix is deprecated; Function `plot_confusion_matrix` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: ConfusionMatrixDisplay.from_predictions or ConfusionMatrixDisplay.from_estimator.\n",
      "  warnings.warn(msg, category=FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 2 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU4AAAEGCAYAAAAKWHxoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAheklEQVR4nO3de5gV1Z3u8e/bTXNR7lcRUFHQSExEQxSjcRByBMnMQHIcrxOJo1EnkpiJMxM1c6LHjDlxMtGJUTGYcKKJ8RIvERMVL9GoGUHwEhUQJSgBRBGam4BAd//mj6rGTdOXXbh3N3v3+3meerr2qlVVq3rDr9eqVbWWIgIzM8tfRVsXwMys1Dhwmpll5MBpZpaRA6eZWUYOnGZmGXVo6wJk1bd3ZRwwpKqti2EZvP7yXm1dBMtoI2tXR0S/3d1//Al7x5rq2rzyPv/y1lkRMWF3z9UWSi5wHjCkiudmDWnrYlgG4/cd2dZFsIwei7uXfpT9V1fXMmfW4LzyVg38c9+Pcq62UHKB08xKQVAbdW1diKJx4DSzggugjvJ9ucaB08yKog7XOM3M8hYE291UNzPLXwC1bqqbmWXje5xmZhkEUFvGI685cJpZUZTvHU4HTjMrgiB8j9PMLIsI2F6+cdOB08yKQdSiti5E0ThwmlnBBVDnGqeZWTaucZqZZZA8AO/AaWaWtwC2R/mOk+7AaWYFF4jaMp5gwoHTzIqiLsq3qV6+fxLMrM3U3+PMZ2mJpCGSnpC0QNJ8SRel6VdIWiHppXSZmLPPpZIWS1okaXxO+oQ0bbGkS3LSh0qak6bfKaljc2Vy4DSzIhC1UZHXkoca4OKIGAGMBi6UNCLddm1EjEyXBwHSbacBHwcmADdKqpRUCdwAnASMAE7POc7V6bGGAWuBc5orkAOnmRVcMgJ8RV5Li8eKWBkRL6TrG4GFwKBmdpkE3BERWyPiTWAxcFS6LI6IJRGxDbgDmCRJwFjg7nT/W4DJzZXJgdPMCi5CbIvKvJYsJB0AHAHMSZOmSnpZ0gxJvdK0QcCynN2Wp2lNpfcB1kVETYP0JjlwmllR1KG8FqCvpHk5y3mNHU9SV+Ae4BsRsQGYBhwEjARWAj9snStzr7qZFUHSOZR3vWx1RIxqLoOkKpKgeVtE3AsQEe/mbL8Z+G36cQWQO4f44DSNJtLXAD0ldUhrnbn5G+Uap5kVQeE6h9J7kD8DFkbENTnpA3OyfQF4NV2fCZwmqZOkocBw4DlgLjA87UHvSNKBNDMiAngCODndfwpwf3Nlco3TzAquvnOoQI4FvgS8IumlNO0ykl7xkenp3gLOB4iI+ZLuAhaQ9MhfGBG1AJKmArOASmBGRMxPj/ct4A5J/w68SBKom+TAaWZFUVugB+Aj4hlo9IHPB5vZ5yrgqkbSH2xsv4hYQtLrnhcHTjMruEBsj/INL+V7ZWbWZjJ2DpUcB04zK7hABWuq74kcOM2sKArYObTHceA0s4KLIN/30EuSA6eZFVzSOZTtdcpS4sBpZkXhziEzswwClfVAxg6cZlYUrnGamWWQzKvuwGlmlkF+02KUKgdOMyu4ZHpg96qbmeUtQm6qm5ll5QfgzcwySMbj9D1OM7MM5BqnmVkWyeNIrnGameXN76qbme0GDytnZpZBMqycm+pmZpn4HqeZWQbJ6EhuqpuZ5S155dKB0/KwakUVP7hoP9a9VwUKJv79Gr5w7mquOn9/lv+5MwCbNlSyd/dapj22iN/f24tf39h/x/5vLuzMDbNe56DDtnDZGQdSvaqK2ho47OhNTP3ecior4eYr92X2o92p6hgM3H8rF1+7jK49atvqksvON6/5C0d/biPrVnfg/LGHANCtZw2X3bSUAYO38e7yjlx1/v68v74DQ4Z9wDevWcawT2zhlqv34e6bPvwuv/CV9zjpjDVEiDdf68wP/2kI27eWbyDZVXnXOIt6ZZImSFokabGkSxrZ3knSnen2OZIOKGZ5iq2yQ3Ded97m5j+8xo9++wYP/LwvS1/vxLd/spRpjy1i2mOLOPbz6zh24joAxn5x7Y70f/3xUvbZbxsHHbYFgG//5C1uemwR059YxPo1HXj6gZ4AHHn8RqY/8Ro3Pb6IQQdu5Y4f92+iNLY7HrmzN98+c+hOaadMXcWLz3TlH447lBef6cqpU1cBsGFtJdP+zyDuuanfTvn77LOdyeesZupJB3P+2EOorAjGTFrXWpewx6hDeS2lqGiBU1IlcANwEjACOF3SiAbZzgHWRsQw4Frg6mKVpzX0GVDD8E8mgW+vrnUMGbaV1SurdmyPgKdm9uSEyWt32feJ3/TiryZ9mL53tzoAamugZpuo//f1qTEbqUzbCYd+avNOx7eP7tU5Xdm4dueG2DHjN/DYXb0BeOyu3hwzYQMA69dU8fqf9qKmZtf//JUdgk6d66ioDDp1qWPNu+3re6rvVc9nKUXFrHEeBSyOiCURsQ24A5jUIM8k4JZ0/W5gnKTS/E028M6yjvz51S587MjNO9JenbM3vfrVMOjAbbvkTwLqup3SLjv9QE795GF06VrHZ/963S77zLq9N58eu7HQRbcGevXdTvWqJPBVr+pAr77bm82/5p0q7p7Wj1/MXcjtL81n08ZKXvhDt9Yo6h6lLiryWkpRMUs9CFiW83l5mtZonoioAdYDfRoeSNJ5kuZJmvfemj3/ft6WTRV899wDuODKFTtqjpDUKsc0Utt87YW96NSljgM+9sFO6d+7fQm3vzif7dvES8903Wnbr340gMoOwdgv7no8KyYRLdSSuvao4ZjxG5hy9KGcccTH6bxXXbv7nurnHMpnKUUlEe4jYnpEjIqIUf367NmvcdVsh++eewBjv7iW4yau35FeWwN/fLAHf/W363bZ58n7ezYaUAE6dg6OGb+eZ2f12JH2yJ29ee6x7nzr+qWUR/18z7Z2dRW9+ye1zN79t7NuTfN9qkd89n3eWdaR9dUdqK0Rf3ywByNGbWqNou4xAqiJiryWUlTMUq8AhuR8HpymNZpHUgegB7CmiGUqqgi45uL9GDJ8K//7/Pd22vbC090YMmwr/fbduZlXVwdPPdBzp86DLZsqWPNu8p+ztgaee6w7Q4ZtBWDuE9349Y39ueLnS+i8VxT3ggyA2Y9053OnVAPwuVOqeXZW92bzr1pRxaFHbqJTlzogGHnc+/xlcadWKOmepZyb6sV8HGkuMFzSUJIAeRpwRoM8M4EpwLPAycDvI6Jko8H85/bm8bt7M/TQLfzj55JHWc6+9G2OGreRP9zfeDP9ldld6bfvdgbu/+F9zw82V3DFlw9k+zZRVweHf+Z9/vqs1QDc8O3BbN8qLj11GAAf+9QmLrp6eStcXftwyY1L+eQx79Ojdw2/nLeAX/xwAHde359v37SUCadVs2pF8jgSQK9+2/nxQ2+wV7daog4mn7ua88YcwqIX9+bp3/XkhlmvU1sjFr/ahYd+ucsdqPJWws3wfKiYcUrSROC/gEpgRkRcJelKYF5EzJTUGfgFcARQDZwWEUuaO+aowzvHc7OGNJfF9jDj9x3Z1kWwjB6Lu5+PiFG7u3+vj/WPsTNOzivvvcdO+0jnagtFrSdHxIMRcXBEHBQRV6Vp34mImen6BxHxdxExLCKOailomlnpKFTnkKQhkp6QtEDSfEkXpem9JT0q6Y30Z680XZKuS58Pf1nSkTnHmpLmf0PSlJz0T0l6Jd3nupae7inNGwxmtkerH8i4QL3qNcDFETECGA1cmD4TfgnweEQMBx5PP0Py7PjwdDkPmAZJoAUuB44meVzy8vpgm+b5Ss5+E5orkAOnmRVcIGrqKvJaWjxWxMqIeCFd3wgsJHmUMfc58FuAyen6JODWSMwGekoaCIwHHo2I6ohYCzwKTEi3dY+I2Wkfy605x2qU31U3s6LI8DplX0nzcj5Pj4jpjWVMX8s+ApgDDIiIlemmd4AB6XpTz5A3l768kfQmOXCaWeFFpvE4V+fTOSSpK3AP8I2I2JB7GzIiQlKrPZHjprqZFVyB73EiqYokaN4WEfemye+mzWzSn6vS9KaeIW8ufXAj6U1y4DSzoihgr7qAnwELI+KanE31z4GT/rw/J/2stHd9NLA+bdLPAk6U1CvtFDoRmJVu2yBpdHqus3KO1Sg31c2s4AJRm0fHT56OBb4EvCLppTTtMuD7wF2SzgGWAqek2x4EJgKLgc3A2QARUS3puyQv5wBcGRHV6fpXgZ8DXYCH0qVJDpxmVhSFGmszIp6BJg82rpH8AVzYxLFmADMaSZ8HHJZvmRw4zazgIlvnUMlx4DSzomhp+L1S5sBpZkVQ3oN8OHCaWVG4xmlmlkEE1NY5cJqZZVKqM1jmw4HTzAoucFPdzCwjdw6ZmWVWupPgtMyB08yKwk11M7MMkl718h1DyIHTzIrCTXUzs4zcVDczyyCQA6eZWVZl3FJ34DSzIggIv3JpZpaNm+pmZhm1y151ST+mmdsUEfH1opTIzEpee35XfV4z28zMmhZAewycEXFL7mdJe0XE5uIXyczKQTk31Vt8J0rSMZIWAK+lnw+XdGPRS2ZmJUxEXX5LKcrnZdL/AsYDawAi4k/A8UUsk5mVg8hzKUF59apHxDJpp78MtcUpjpmVhWi/nUP1lkn6DBCSqoCLgIXFLZaZlbwSrU3mI5+m+gXAhcAg4G1gZPrZzKwZynMpPS3WOCNiNXBmK5TFzMpJXVsXoHjy6VU/UNIDkt6TtErS/ZIObI3CmVmJqn+OM5+lBOXTVP8VcBcwENgX+DVwezELZWalLyK/pRTlEzj3iohfRERNuvwS6FzsgplZiWuPjyNJ6p2uPiTpEuAOkss8FXiwFcpmZqWsRJvh+Wiuc+h5kkBZf/Xn52wL4NJiFcrMSp9KtDaZjyab6hExNCIOTH82XNw5ZGZNC0FdnksLJM1IO6ZfzUm7QtIKSS+ly8ScbZdKWixpkaTxOekT0rTFaSu6Pn2opDlp+p2SOrZUprzm75R0mKRTJJ1Vv+Szn5m1Y4W7x/lzYEIj6ddGxMh0eRBA0gjgNODj6T43SqqUVAncAJwEjABOT/MCXJ0eaxiwFjinpQLl8zjS5cCP0+UE4D+Av21pPzNr5woUOCPiKaA6z7NOAu6IiK0R8SawGDgqXRZHxJKI2EbSZzNJybvkY4G70/1vASa3dJJ8apwnA+OAdyLibOBwoEeeF2Fm7VX+gbOvpHk5y3l5nmGqpJfTpnyvNG0QsCwnz/I0ran0PsC6iKhpkN6sfALnloioA2okdQdWAUPy2M/M2qtsD8CvjohROcv0PM4wDTiI5BXwlcAPi3YtjchnkI95knoCN5P0tL8PPFvMQplZ6Stmr3pEvLvjPNLNwG/TjyvYuWI3OE2jifQ1QE9JHdJaZ27+JrVY44yIr0bEuoi4CfhfwJS0yW5m1rQiPgAvaWDOxy8A9T3uM4HTJHWSNBQYDjwHzAWGpz3oHUk6kGZGRABPkNySBJgC3N/S+Zt7AP7I5rZFxAstHdzM2q9C1Tgl3Q6MIbkXuhy4HBgjaSRJ6H2L9DnziJgv6S5gAVADXBgRtelxpgKzgEpgRkTMT0/xLeAOSf8OvAj8rKUyNddUb+6eQZD0RLW6Nxb15PPHTW6LU9tue6utC2BtoUBvDkXE6Y0kNxncIuIq4KpG0h+kkbceI2IJSa973pqbrO2ELAcyM9uhhN9Dz0deU2eYmWXmwGlmlo3KeCBjB04zK44yrnHm88qlJP29pO+kn/eTlOlGqpm1L4r8l1KUz5tDNwLHAPU9WxtJXpY3M2taGU+dkU9T/eiIOFLSiwARsTafYZfMrJ0r0dpkPvIJnNvTIZkCQFI/ynr+OjMrhFJthucjn8B5HXAf0F/SVSSvJv1bUUtlZqUt2nmvekTcJul5kqHlBEyOiIVFL5mZlbb2XOOUtB+wGXggNy0i/lLMgplZiWvPgRP4HR9O2tYZGAosIhma3sysUe36HmdEfCL3czpq0leLViIzsz1c5jeHIuIFSUcXozBmVkbac41T0jdzPlYARwJvF61EZlb62nuvOtAtZ72G5J7nPcUpjpmVjfZa40wffO8WEf/cSuUxszIg2mnnUP3kRZKObc0CmVmZaI+Bk2SCoyOBlyTNBH4NbKrfGBH3FrlsZlaqSnjko3zkc4+zM8kUmmP58HnOABw4zaxp7bRzqH/ao/4qHwbMemX8t8TMCqG91jgrga7sHDDrlfGvxMwKooyjRHOBc2VEXNlqJTGz8tGOZ7kszaGZzWyP0F6b6uNarRRmVn7aY+CMiOrWLIiZlZf2/sqlmVk27fgep5nZbhHl3UniwGlmxeEap5lZNu21V93MbPeVceCsaOsCmFkZSgcyzmdpiaQZklZJejUnrbekRyW9kf7slaZL0nWSFkt6OZ3qp36fKWn+NyRNyUn/lKRX0n2uk9Ti7VkHTjMrjshzadnPgQkN0i4BHo+I4cDj6WeAk4Dh6XIeMA2SQAtcDhwNHAVcXh9s0zxfydmv4bl24cBpZkWhyG9pSUQ8BTR8rnwScEu6fgswOSf91kjMBnpKGgiMBx6NiOqIWAs8CkxIt3WPiNkREcCtOcdqku9xmllxFPce54CIWJmuvwMMSNcHActy8i1P05pLX95IerMcOM2sKDL0qveVNC/n8/SImJ7vzhERUuv24TtwmlnhBVkGMl4dEaMynuFdSQMjYmXa3F6Vpq8AhuTkG5ymrQDGNEh/Mk0f3Ej+Zvkep5kVXP1kbYW4x9mEmUB9z/gU4P6c9LPS3vXRwPq0ST8LOFFSr7RT6ERgVrptg6TRaW/6WTnHapJrnGZWHAVqPEu6naS22FfScpLe8e8Dd0k6B1gKnJJmfxCYCCwGNgNnQzJokaTvAnPTfFfmDGT0VZKe+y7AQ+nSLAdOMysKRWEiZ0Sc3sSmXYa+THvGL2ziODOAGY2kzwMOy1ImB04zKzyPjmRmlp3fVTczy8gDGZuZZeUap5lZBh/tUaM9ngOnmRWHA6eZWf7qH4AvVw6cZlYUqivfyOnAaWaF5+c4bXdVdazl6uufoapjHZWVwR+f2JfbZnwMCM46byHHnfA2dbXid785gAfuPojRx63k7899jQiorRXTr/sEC17uQ78Bm/m37z1HRUVQ2SF44O6hPHT/0La+vLI3+Zz3OOnMaqTgodv6cN9P+3HgiC187fvL6bJ3He8u78jVF+7H5vcrATh16rtMOL2a2jox7d/25fk/dG/jK2hbfhxpN0gaQjIo6ACSvz3TI+JHDfII+BHJu6WbgS9HxAvFKlNr276tgssuOpYPtnSgsrKOH0x7mnlz+jNk/4307b+F888YR4To0XMrAC8934/Zz+wDiAMOWs8lV87jgjPHsXZNZy6+4LPUbK+kc5cabrz198x5Zh+q13Rp2wssY/sfsoWTzqzm658fzvZt4nu/WsKcx7rzjf9cxs1X7ssrs7ty4mlrOPkfV3HrDway3/APGDNpHeedcAi9B2zn+3cu4ZzjulFXV86T5LagjGucxRwdqQa4OCJGAKOBCyWNaJCn0WHuy4f4YEvyt6lDh6TWScDEyW9x+/8/hIjkP9X6dZ0A0rxJWufOtTv+4dXUVFCzPanVVFXVIY9pVXT7Dd/Kay/uxdYtFdTVipef7cqxE9cz+MCtvDJ7bwBefKobx31+PQDHjF/Pk/f3ZPu2Ct5d1om33+rIIUdsbstLaHNFHh2pTRWtxpkO17QyXd8oaSHJyMoLcrLtGOYemC2pZ/0Ye8UqV2urqAh+9LMnGThoE7+7byiLFvRm4KBNHD9uBcccv5L16zrxk//6BG8v7wrAMce/zZTzF9Kz11au+JfRO47Tt/8WrviP2QwcvIkZN45wbbPI3nqtM1/+1kq69aph2wcVfHrsBt54uQtLX+/MMRM28OzDPfjsX6+n377bAeg7cDsLn997x/6rV3akzz7b26r4bS+AAg3ysSdqlbqLpAOAI4A5DTY1NZx9w/3PkzRP0rxttaX1V7yuTnzt7BOY8sXxHHzoOvYfuoGqqjq2bavkG+eOYdbM/bno0hd35H/2qX254MxxfPfSo/jSVxbuSF+9qgtTv3wCXzl1HOMmLKNnrw/a4nLajWWLO3PXjf35f7cv4arblrBkfhfqasU13xzC30xZzfUPv06XrrXUbGvHTfEWFGqWyz1R0QOnpK7APcA3ImLD7hwjIqZHxKiIGNWxcq/CFrCVbHq/ipdf6MunRq9i9Xtd+O8/DATgv58ayNCDdv21zP9TX/bZdzPde2zdKb16TReWvtmdjx/ecO4qK7RZt/dh6oSD+ecvDuP99ZUsX9KJZYs7c9npBzF1wsE8+ZterFzaEYDVK6vot++2Hfv2HbiNNe9UtVXR21wrDGTcpooaOCVVkQTN2yLi3kayNDXMfVno3nMre3dNmmsdO9Yy8tOrWLa0K7Of3odPHrkagE8csYYVy5Jm+sBB71N/Y/Ogg9fRoaqODes70qffFjp2rAWga7dtfPyTa1j+l66tf0HtTI8+yXfXb9A2jp24nifu67UjTQrOuOhdfvuLPgDMfqQHYyato6pjHQOGbGXQ0G0serE0/8gXRET+SwkqZq+6gJ8BCyPimiayzQSmSrqDZL7j9eV0f7N3nw/45rdfpKIiUEXwzO8HMfe/92HBy334l+88z+RT/syWLR247uqRABw7ZiVjJyyjtkZs3VrJ1ZePAsSQ/Tdy7tT5BMlf8ntvH8bSJe37UZfW8J2fLqVbrxpqt4vrLxvEpg2VTD6nmr/5cvJH748P9eCRO3oDsPT1zjz1QE+mP7mI2tokf7vuUad0a5P5UBQp4ks6DngaeIUPp226DNgPICJuSoPr9SQTwG8Gzk5HY25Sj877xGcGf6koZbbiqFnyVlsXwTJ6LO5+fjcmUNuhW8/BccTxF+WV9+kH/vUjnastFLNX/Rnqn61pOk+Tw9ybWWkr5xqn3xwys8ILoLZ8I6cDp5kVhWucZmZZlWiPeT4cOM2sKFzjNDPLwsPKmZllI0DuHDIzy0a+x2lmloGb6mZmWZXue+j5cOA0s6Jwr7qZWVaucZqZZRDuVTczy65842brTJ1hZu2PIvJa8jqW9JakVyS9JGlemtZb0qOS3kh/9krTJek6SYslvSzpyJzjTEnzvyFpyu5emwOnmRVH4UeAPyEiRuaM3XkJ8HhEDAceTz9DE7PnSuoNXE4yaPpRwOX1wTYrB04zK7wgGb48n2X3TQJuSddvASbnpN8aidlAT0kDgfHAoxFRHRFrgUdJBlHPzIHTzApO5NdMT5vqfetnsU2X8xo5ZACPSHo+Z/uAnKl23gEGpOtNzZ6b16y6+XDnkJkVR13e1cnVeUydcVxErJDUH3hU0mu5GyMipNZ7ctQ1TjMrvAI31SNiRfpzFXAfyT3Kd9MmOOnPVWn2pmbPLdisug6cZlYUhepVl7S3pG7168CJwKsks+TW94xPAe5P12cCZ6W966P5cPbcWcCJknqlnUInpmmZualuZsVRuDeHBgD3JZPi0gH4VUQ8LGkucJekc4ClwClp/geBicBi0tlzk+JEtaTvAnPTfFdGRPXuFMiB08yKoHCDfETEEuDwRtLXAOMaSW9y9tyImAHM+KhlcuA0s8LzLJdmZtl5IGMzs6wcOM3MMgigzoHTzCwDjwBvZpadA6eZWQYB1H60ETz2ZA6cZlYEAeHAaWaWjZvqZmYZuFfdzGw3uMZpZpaRA6eZWQYRUFvb1qUoGgdOMysO1zjNzDJy4DQzyyLcq25mlklA+AF4M7OM/MqlmVkGEVmmBy45DpxmVhzuHDIzyyZc4zQzy8IDGZuZZeNBPszMsgkg/MqlmVkG4YGMzcwyCzfVzcwyKuMap6LEer4kvQcsbetyFEFfYHVbF8IyKefvbP+I6Le7O0t6mOT3k4/VETFhd8/VFkoucJYrSfMiYlRbl8Py5++s/apo6wKYmZUaB04zs4wcOPcc09u6AJaZv7N2yvc4zcwyco3TzCwjB04zs4wcOFuZpAmSFklaLOmSRrZ3knRnun2OpAPaoJiWkjRE0hOSFkiaL+miRvJI0nXpd/aypCPboqzWehw4W5GkSuAG4CRgBHC6pBENsp0DrI2IYcC1wNWtW0proAa4OCJGAKOBCxv5zk4ChqfLecC01i2itTYHztZ1FLA4IpZExDbgDmBSgzyTgFvS9buBcZLUimW0HBGxMiJeSNc3AguBQQ2yTQJujcRsoKekga1cVGtFDpytaxCwLOfzcnb9T7gjT0TUAOuBPq1SOmtWetvkCGBOg035fK9WRhw4zfIgqStwD/CNiNjQ1uWxtuXA2bpWAENyPg9O0xrNI6kD0ANY0yqls0ZJqiIJmrdFxL2NZMnne7Uy4sDZuuYCwyUNldQROA2Y2SDPTGBKun4y8PvwWwptJr2//DNgYURc00S2mcBZae/6aGB9RKxstUJaq/N4nK0oImokTQVmAZXAjIiYL+lKYF5EzCT5T/oLSYuBapLgam3nWOBLwCuSXkrTLgP2A4iIm4AHgYnAYmAzcHbrF9Nak1+5NDPLyE11M7OMHDjNzDJy4DQzy8iB08wsIwdOM7OMHDjLkKRaSS9JelXSryXt9RGO9XNJJ6frP21kgIvcvGMkfWY3zvGWpF1mRGwqvUGe9zOe6wpJ/5y1jGa5HDjL05aIGBkRhwHbgAtyN6ZvJGUWEedGxIJmsowBMgdOs1LjwFn+ngaGpbXBpyXNBBZIqpT0A0lz0zEkz4cdY0ten44Z+hjQv/5Akp6UNCpdnyDpBUl/kvR4OgDGBcA/pbXdz0rqJ+me9BxzJR2b7ttH0iPp+JY/BVoc/UnSbyQ9n+5zXoNt16bpj0vql6YdJOnhdJ+nJX2sIL9NM/zmUFlLa5YnAQ+nSUcCh0XEm2nwWR8Rn5bUCfijpEdIRv85hGS80AHAAmBGg+P2A24Gjk+P1TsiqiXdBLwfEf+Z5vsVcG1EPCNpP5I3pg4FLgeeiYgrJX2eZAzSlvxDeo4uwFxJ90TEGmBvkreu/knSd9JjTyWZSO2CiHhD0tHAjcDY3fg1mu3CgbM8dcl5PfBpktc4PwM8FxFvpuknAp+sv39JMpjIcOB44PaIqAXelvT7Ro4/Gniq/lgRUd1EOT4HjMgZTrR7OsrQ8cAX031/J2ltHtf0dUlfSNeHpGVdA9QBd6bpvwTuTc/xGeDXOefulMc5zPLiwFmetkTEyNyENIBsyk0CvhYRsxrkm1jAclQAoyPig0bKkjdJY0iC8DERsVnSk0DnJrJHet51DX8HZoXie5zt1yzgH9Mh05B0sKS9gaeAU9N7oAOBExrZdzZwvKSh6b690/SNQLecfI8AX6v/IGlkuvoUcEaadhLQq4Wy9iCZTmRzeq9ydM62CpJRpEiP+Uw6Xuabkv4uPYckHd7COczy5sDZfv2U5P7lC5JeBX5C0gK5D3gj3XYr8GzDHSPiPZK5de6V9Cc+bCo/AHyhvnMI+DowKu18WsCHvfv/lyTwzidpsv+lhbI+DHSQtBD4PkngrrcJOCq9hrHAlWn6mcA5afnms+sUJWa7zaMjmZll5BqnmVlGDpxmZhk5cJqZZeTAaWaWkQOnmVlGDpxmZhk5cJqZZfQ//uXFVoD6DcoAAAAASUVORK5CYII=\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.9/dist-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function plot_roc_curve is deprecated; Function :func:`plot_roc_curve` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: :meth:`sklearn.metric.RocCurveDisplay.from_predictions` or :meth:`sklearn.metric.RocCurveDisplay.from_estimator`.\n",
      "  warnings.warn(msg, category=FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA5GklEQVR4nO3deXxU5fX48c9JSEhCCFsAgRB2BVmFAK64oHVBcQEE1FZbl1q3Li5V2ypi219bl6rVoqJ+RYvggiguhSogqMhO2EHZCWEJECAhZD+/P+5NHEKSuSGZTCZz3q/XvDJ3m3vuJLnn3ud57vOIqmKMMSZ8RQQ7AGOMMcFlicAYY8KcJQJjjAlzlgiMMSbMWSIwxpgw1yDYAVRVYmKiduzYMdhhGGNMSFm2bNl+VW1Z3rKQSwQdO3Zk6dKlwQ7DGGNCiohsr2iZFQ0ZY0yYs0RgjDFhzhKBMcaEOUsExhgT5iwRGGNMmAtYIhCRN0Rkn4isqWC5iMgLIrJJRFaJSP9AxWKMMaZigbwjeBO4rJLllwPd3NcdwIQAxmKMMaYCAXuOQFXni0jHSla5GnhLnX6wF4pIUxFpo6q7AxWTMcbUFUXFyrGCIo7lu6+CInLyC3+cV1BETv7x74d2b0Xf9k1rPJZgPlDWDtjpM53mzjshEYjIHTh3DSQnJ9dKcMaY8JZfWFzmpFxIbjkn55L3x/Ld6YIijrkn9Jz8oh+3KXOCzy8srnJMrRo3rHeJwDNVfRV4FSAlJcVG0jHGnKCoWMnOLeRIbgFZuYVkuT/3HMnl4NF8n5NyIccKio87WfuezEveFxZX7VQTIRAX3YCYqEjiop1XyftTEqKIjY4k1p2OiY4kLqoBsdERxEY3KJ3vu05slO90Axo2iCAiQgLy3QUzEewC2vtMJ7nzjDGGgqJi9h7JZffhXNIPHSP9UC67Dx8jM6eg9CT/489CsvMKK/28qEgpPbnGuSff2OhI4hs2IDG+YTkn35ITeYPjTuqxPuv5bhMdGYFIYE7UgRbMRDADuEdEpgKDgcNWP2BM/VZUrBw5VkBmTj6ZOQUc8vm5PzuftMyc0pP+vqxcyl6UJ8Q0oHmjaBrHRNE4pgGJiY1IiIkqnW4c04CEmCgSYhuUzmsSG0XbprFERVpr+YoELBGIyBTgAiBRRNKAx4EoAFV9GfgcuALYBOQAPw9ULMaYwMrKLWDHwRx2Hsxhx8Ec9h3JIzOn5ISfzyH3/eFjBVQ0THqDCCGpWSxtm8ZyTtdE2jWNoU1TZ7ptE+d9fMOQKM0OOYFsNTTWz3IF7g7U/o0x1ZdXWMSew7nsz84jIyuPjOx89mflsT87j5VphyguprQM3ldsVCTNG0XTNC6KZnHRtGsaS7O4aJrFRdE0LppmjdyfPvMSYhqEbNFKqLP0aowhK7eALRlH2ZyRzaq0w2zOyGbr/qPsOnTshCt4EWgWF01ifDR5hcVc2vMUkpvH0aFFHMnN42jfPI4msVHBORBzUiwRGBNGVJXDxwrYfTiX1J2HmP99BvO+zyAnv6h0ndioSLq1jmdAh2aMHJBEUrM4EuOjadm4IS3jG9K8UTQNrLy9XvGbCEQkCRgDnAe0BY4Ba4DPgP+qatUbwxpjAkZV2ZeVx6Z92ce9th84SkZ2HgVFP17iJ8ZHc3qbBAZ0aEb/Ds3o0rIRHVo0sorVMFNpIhCR/8N5yOtT4O/APiAGOBWn+4g/iMjDqjo/0IEaY45XVKzsOJjD5n3ZbMr48YS/eV82WT5NKRs3bECXVvGc2bkFrRJinCv7xg3p3a4JHVvEWbm88XtH8Iyqltdp3BrgQxGJBuxRX2NqQXGxkpp2iLkb9jHv+ww27M4iv+jHG/KWjRvStWU815zRjq6t4ktfrRo3tJO9qVSlicA3CYhIc3feQZ/l+TjNP40xAbLzYA7/WbSd/3y3naP5RUQI9E9uxi3ndKRry3i6uCd8q6A1J8tf0VAy8A9gKHDImSUJwBzgYVXdFugAjQk3uQVFpO48xMItB/hu8wEWbXWuvS48rSX9k5vx07M60DQuOshRmvrEX9HQu8BzwI2qWgQgIpHAKGAqcGZAozOmnlNVFm09yJpdh1mbfoS16YfZnHGUomJFBHq1bcLt53Xi6n7t6NWuSbDDNfWUv0SQqKrv+s5wE8JUEXkycGEZUz/5nviX78hk6bZM9mXlAdA6oSE92zbh0p6n0CepKYM6NqdJnBX3mMDzlwiWici/gUn82GV0e+BmYEUgAzOmvjiSW8CX6/byxbq9LN+Ryd4jzok/qVksZ3VpwYAOzbjwtFa0bx4X5EhNuPKXCH4G3Ao8gdOMFJxxAz4BXg9gXMaEtMKiYj5Zlc7nq/cwb2MG+UXFtGkSQ0rH5gzplsg5XRNJamYnflM3+Gs1lI8zhKQNI2mMR99tPsCj01ezdf9RAG45uyPD+7XljPZNrRmnqZNOuosJEblSVT+tyWCMCUV5hUUs257J1z/sZ97GDNbtPkLrhIb8a+wZXN7rFOuOwdR51elraCDOE8fGhJXcgiLmbNjH2vTDpO48xLebDgBON8r9OzRj3FWnM2ZQMjFRkUGO1BhvTjoRqOrjNRmIMXVddl4hE77axLtL0tifnUdkhNCtVTwX92jNpT1bc3nvNtZfvglJXjqdSwBaqurmMvP7qOqqgEVmTB2gqmzYk8WMlelMWbyDQzkF9GybwN9H9Oacrol21W/qBX9PFl+P80DZPhGJAm5R1SXu4jeB/gGNzpgg2X34GJMWbGf2+r38sC+byAjh/FNbcs9FXemf3CzY4RlTo/zdETwKDFDV3SIyCHhbRB5R1emANX8w9c4Pe7N4ed4WPk7dRWGx0rxRNOOuOp2r+ralRXzDYIdnTED4SwSRJQPKq+piEbkQ+FRE2gMVjDxqTGgpLlYWbzvIa19v5cv1e4mNiuSmMzvw83M60qFFo2CHZ0zA+UsEWSLSpaR+wL0zuAD4COgZ2NCMCZzDxwqYsTKdOev3Mv+H/RQVK9GREfx6aDduPrsjzRtZp24mfPhLBL+iTBGQqmaJyGXA9QGLypgAUFXmfZ/B9BW7mLlmD3mFxXRObMT1KUl0a9WY6we2t1Y/Jiz5e7J4ZQXzC4DJAYnImBq2Ykcmby/czvzvM9ifnQ/ATWcmMzolmV7tEuxpXxP27PLH1EtZuQW8s2gHH6ems273ERrHNGBo91ac0zWR809tSauEmGCHaEydYYnA1Cu7Dh3j+S+/Z8bKdHILnGEcn7y6J9f2T7JiH2MqYP8ZJuRl5xUya80ePkrdxTeb9hMVEcHIlCRG9E+if7J19GaMP54TgYiMU9VxFU0bU5tUla9/2M+05WnMWruH3IJikprFctcFXRgzMNn69jemCqpyR7DMz7QxtWLnwRwenb6ar3/YT3zDBozon8R1/dvRP7mZXf0bcxI8JwJV/aSyaWMC7fCxAl7/egsTv95KhMAfh/Xgp2d1oGED6+/HmOrw19fQv6jkCWJVva/GIzKmjG37j/Li3E18sjKdvMJihvVuwx+G9aBt09hgh2ZMveDvjmBprURhTDmycgv46+freW9pGg0ihJEDkrhhcDI92zYJdmjG1Cv+Hiib5DstInGqmhPYkEy423cklxfm/MDMNXs5eDSPMYOS+c3Qbtb235gA8VRHICJn4QxWHw8ki0hf4Jeqepef7S4DngcigddU9W9llicDk4Cm7joPq+rnVT0IUz8cyS3gPwu3M3H+FnLyizj/1JbceUEX6/bZmADzWln8HHApMAOcridEZEhlG4hIJPAScAmQBiwRkRmqus5ntT8C76nqBBE5Hfgc6FilIzAhr7hYeeu7bTz9v+/JzivkvG6JPH5VT7q2ig92aMaEhaq0GtpZpmlekZ9NBgGbVHULgIhMBa4GfBOBAgnu+yZAutd4TOgrLlZmb9jHU7M28P3ebM4/tSUPXnoavdpZHYAxtclrItgpImcD6o5U9mtgvZ9t2gE7fabTgMFl1hkH/E9E7gUaAReX90EicgdwB0BycrLHkE1dpKrMXLOHKUt2krojkyO5hXRObMQLY8/gqj5t7DkAY4LAayK4E6esvx3OVfss4O4a2P9Y4E1Vfcath3hbRHqparHvSqr6KvAqQEpKig2IE6K27j/K7z9YxeJtB+nQIo5hfdpwZucWXNG7DVGREcEOz5iw5SkRqOp+4MYqfvYuoL3PdJI7z9etwGXuPr4TkRggEdhXxX2ZOiy3oIjXv9nK81/+QHSDCG47txMPX96dBnbyN6ZO8NpqqDPOHcGZOOX63wG/LSn/r8ASoJuIdMJJAGOAG8qsswMYCrwpIj2AGCCjSkdg6rS5G/bx0LRVZGTlccnprfnLNb2sGagxdYzXoqF3cFoAXetOjwGmcGKZfylVLRSRe3CKkSKBN1R1rYiMB5aq6gzgfmCiiPwWJ8HcoqpW9FMP5BcW89LcTbw4dxOntm7M82P6cXaXxGCHZYwph3g574rIKlXtU2beSlXtG7DIKpCSkqJLl9oDz3XZt5v289jHa9iccZRr+rVl/DW9SIiJCnZYxoQ1EVmmqinlLfPX11Bz9+1/ReRhYCrOlftonDb/xpTauv8of/lsHV+u30ebJjG89rMULj69dbDDMsb44a9oaBnOib+kTd8vfZYp8EgggjKh5VBOPi/O2cRb322noVsZfN/F3ewuwJgQ4a+voU61FYgJTbPX7+XhD1dzIDuPa89I4veXnWaVwcaEmKqMUNYLOB2nZQ8AqvpWIIIydd+x/CL+MH01H67YRfdTGvN/twy0J4KNCVFem48+DlyAkwg+By4HvgEsEYShnQdz+OXby1i/5wj3XdSVey7qRnQDeybAmFDl9Y5gJNAXWKGqPxeR1sB/AheWqauWbT/IL99eTl5hEW/cPJALu7cKdkjGmGrymgiOqWqxiBSKSALOk7/t/W1k6pdPVqZz//srOSUhhnduH8yprRsHOyRjTA3wmgiWikhTYCJOS6JsnKeLTRjILyzm8RlrmLJ4J73aJfDWLwbTvFF0sMMyxtQQr30NlQxA87KIzAQSVHVV4MIydYGqsnrXYR75cDVr04/wy/M78+uh3YiL9tzGwBgTAvw9UNa/smWqurzmQzJ1wbb9R/nTx2v4+of9NG7YgL9e25sbBlsX4MbUR/4u7Z6pZJkCF9VgLKaOeGfRDh6fsYaoyAj+OKwHowa0p0mcPRxmTH3l74GyC2srEFM3PD1rIy/O3cT5p7bkqZF97OEwY8KAFfYawBk28p9ffs+LczcxZmB7nrymlw0WY0yYsERgyM4r5M63l/HNpv2MTmnPX67tTWSEDRlpTLiwRBDm1u8+wt3vLGf7gRz+cm0vbhiUbOMGGxNmvHYxIThDVXZW1fEikgycoqqLAxqdCZjComI+XLGLv3y2npioCCb9fBDndrOBY4wJR17vCP4NFOO0EhoPZAHTgIEBissEUHZeIbdPWsp3Ww7Qrmksk28bTMfERsEOyxgTJF4TwWBV7S8iKwBUNVNE7NHSEJR+6Bh3vL2U9buz+MfIPozsn0SE1QcYE9a8JoICEYnEeXYAEWmJc4dgQkhRsfLrqSvYtj+HV386gKE9bPQwYwx4bR/4AjAdaCUif8HpgvqvAYvKBMSzX2xkybZMxl/d05KAMaaU176GJovIMmAozrCV16jq+oBGZmrUxj1ZTPhqM6MGJHFd/6Rgh2OMqUO8thp6AZiqqi8FOB4TADsP5nDnf5bRJDaKR6/oEexwjDF1jNeioWXAH0Vks4g8LSIpgQzK1Jz0Q8cY+fICDmTn8drNKTSz7qONMWV4SgSqOklVr8BpLroR+LuI/BDQyEy15RYUcftbS8nJK+LdX57FgA7Ngx2SMaYOqmpnMl2B7kAHYEPNh2Nqyq5Dxxg7cSFr04/w9PV96dEmIdghGWPqKK91BP8ArgU2A+8CT6rqoQDGZaohIyuP0a98x4HsfF6+qT+X9jwl2CEZY+owr88RbAbOUtX9gQzGVF9JcdD+7Dzeuf1M+ic3C3ZIxpg6zt8IZd1VdQOwBEh2+xgqZSOU1S3Fxcr9761kZdohJtzY35KAMcYTf3cEvwPuoPyRymyEsjok82g+D3+4illr9/LI5d25rFebYIdkjAkR/kYou8N9e7mq5vouExEbuqqO2HcklxtfW8S2A0d59Iru3H5e52CHZIwJIV5bDS3wOO84InKZiGwUkU0i8nAF61wvIutEZK2IvOMxHuNKy8zhqhe/YWdmDpN+Pog7hnSx8QSMMVXir47gFKAdECsiZ+B0LwGQAMT52TYSeAm4BEgDlojIDFVd57NON+AR4By3R9NWJ30kYehwTgF3vLWMY/lFfHDn2fRq1yTYIRljQpC/OoJLgVuAJOBZn/lZwKN+th0EbFLVLQAiMhW4Gljns87twEuqmgmgqvs8Rx7mtmRk87M3FrPncC6v/HSAJQFjzEnzV0cwCZgkIiNUdVoVP7sdsNNnOg0YXGadUwFE5FsgEhinqjPLfpCI3IFTaU1ycnLZxWFnf3Yet721lJz8It6/8yzOsNZBxphq8Fc0dJOq/gfoKCK/K7tcVZ8tZ7Oq7r8bcAHOXcd8Eeld9mE1VX0VeBUgJSVFq7nPkJZfWMxdk5eTfugYb/1isCUBY0y1+SsaKhm/MP4kPnsX0N5nOsmd5ysNWKSqBcBWEfkeJzEsOYn9hYXHZ6xl8daDPD2qL4M6Wd9Bxpjq81c09Ir784mT+OwlQDcR6YSTAMYAN5RZ5yNgLPB/IpKIU1S05ST2FRbW7DrM1CU7uPXcTowcYGMKGGNqhqfmoyLyDxFJEJEoEZktIhkiclNl26hqIXAPMAtYD7ynqmtFZLyIDHdXmwUcEJF1wFzgQVU9cPKHU38VFBXz+2mraBYXza8v7hbscIwx9YjXvoZ+oqoPici1wDbgOmA+8J/KNlLVz4HPy8x7zOe94jy9fEL9gznea19vZW36EV684QwSYqKCHY4xph7x+kBZScIYBryvqocDFI8px44DOTw/+3su63kKV/ZpG+xwjDH1jNc7gk9FZANwDPiViLQEcv1sY2pAVm4Bv353BQ0iIhg3vGewwzHG1ENeRyh7GDgbSHFb+BzFeTjMBJCq8siHq1mx4xDPXN+XU5pY907GmJrndWCaKOAmYIjbj8084OUAxmWAqUt28umq3fzuklNtcBljTMB4LRqaAEQB/3anf+rOuy0QQRnYuCeLP320hrM6t+DO87sEOxxjTD3mNREMVNW+PtNzRGRlIAIyjqdmbSQ2OpJ/39if6AZVHVraGGO883qGKRKR0stSEekMFAUmJDNzzW6+XL+XO8/vQrNG0cEOxxhTz3m9I3gQmCsiW3C6ou4A/DxgUYWxzRnZPPD+Kvq2b2oDzBhjaoXfROA2FT2M0610yXgBG1U1L5CBhaOSMYejG0QwwYqEjDG1pNIzjYjcBqwF/gWkAh1VdZUlgcD4YHkaqTsP8cdhPWjbNDbY4RhjwoS/O4LfAD1VNcOtF5gMzAh4VGGouFh545ut9GybwLVntAt2OMaYMOKv7CFfVTMA3JHGGgY+pPA0Yd5mNuzJ4vbzOtuYw8aYWuXvjiBJRF6oaFpV7wtMWOFlbfphnvvye4b1acPV/awvIWNM7fKXCB4sM70sUIGEqx/2ZnHzG4tJiIli/PCedjdgjKl1XsYsNgFyLL+IUa98R1RkBJN+MZAW8VbyZoypff5aDU0UkV4VLGskIr8QkRsDE1r9pqo8NG0Vh3IKeG50P3q2bRLskIwxYcpf0dBLwGMi0htYA2QAMTjjCicAb+C0JDJV9MGyND5Zmc7t53XinK6JwQ7HGBPG/BUNpQLXi0g8kAK0wRmTYL2qbgx8ePVTYVExE77aTO92TXj48h7BDscYE+Y8dTGhqtnAV4ENJXz87b8b2LL/KC/ecAaREVY5bIwJLuvDoJb9sDeL17/dyg2Dk23YSWNMnWCJoJa9Mn8LMQ0ieeAnpwU7FGOMAaqYCEQkLlCBhIPtB44yIzWdUSlJNLfupY0xdYSnRCAiZ4vIOmCDO91XRP7tZzNTxrgZa2kQKdx1Qddgh2KMMaW83hH8E7gUOACgqiuBIYEKqj7aeTCHr3/Yz9AerW0QemNMneK5aEhVd5aZZSOUVcFTszYSESE8ekX3YIdijDHH8ZoIdorI2YCKSJSIPACsD2Bc9cryHZnMWJnOmIHtadPExhkwxtQtXhPBncDdQDtgF9APuCtAMdU742aspXmjaB66zO4GjDF1j9cxi09T1eP6FBKRc4Bvaz6k+mXFjkxWpR3mvou6Et/Q69dtjDG1x+sdwb88zjNlvDJvC1GRwq02EL0xpo6q9BJVRM4CzgZaisjvfBYlAJGBDKw+mLthHzPX7uG+i7rSJDYq2OEYY0y5/JVVRAPx7nqNfeYfAUYGKqj6YsJXm2nTJIZ7LuoW7FCMMaZC/nofnQfME5E3VXV7VT9cRC4Dnse5e3hNVf9WwXojgA+Agaq6tKr7qYvW7DrM4m0H+eOwHkQ3sJ48jDF1l9fayxwReQroiTMeAQCqelFFG4hIJM54BpcAacASEZmhquvKrNcY+DWwqIqx11kFRcU8+MEqGjdswKgB7YMdjjHGVMrrpepknO4lOgFPANuAJX62GQRsUtUtqpoPTAWuLme9J4G/A7keY6nzXpq7ifW7j/C3EX1oEmd1A8aYus1rImihqq8DBao6T1V/AVR4N+BqB/g+jZzmzislIv2B9qr6WWUfJCJ3iMhSEVmakZHhMeTgWJd+hBfnbGJ437YM69Mm2OEYY4xfXhNBgftzt4gME5EzgObV2bGIRADPAvf7W1dVX1XVFFVNadmyZXV2G1CqyqPTV9M0LoonhvcMdjjGGOOJ1zqCP4tIE5yT9r9wmo/+xs82uwDfAvIkd16JxkAv4CsRATgFmCEiw0O1wnjOhn2k7jzE30f0ppl1M22MCRFeh6r81H17GLgQSp8srswSoJuIdMJJAGOAG3w+8zBQOmq7iHwFPBCqSUBVeePbrZySEMN1/ZOCHY4xxnhWadGQiESKyFgReUBEernzrhSRBcCLlW2rqoXAPcAsnA7q3lPVtSIyXkSG11D8dcZrX2/l200HuO28TkRFWnNRY0zo8HdH8DpO8c5i4AURSQdSgIdV9SN/H66qnwOfl5n3WAXrXuAh3jpp2fZM/jZzAxf3aM0vzukU7HCMMaZK/CWCFKCPqhaLSAywB+iiqgcCH1romLJ4B/ENG/DP0X2JiJBgh2OMMVXirwwjX1WLAVQ1F9hiSeB4+YXFfPPDfs7u0oLGMfbMgDEm9Pi7I+guIqvc9wJ0cacFUFXtE9DoQsCcDfvYcySX4X3bBjsUY4w5Kf4SQY9aiSKEvbd0Jy0bN+Ti01sHOxRjjDkp/jqdq3JHc+Fk58Ec5m7cx70XdrWWQsaYkGVnr2p4f+lOBBgzKDnYoRhjzEmzRHCSCouKmbZ8F2d3SaRtUxuQ3hgTujwnAhGJFZHTAhlMKJm7MYNdh45x05kdgh2KMcZUi6dEICJXAanATHe6n4jMCGBcdd5nq9JpEhvF0B6tgh2KMcZUi9c7gnE44wscAlDVVJyxCcLSyp2HmLEyneF921olsTEm5HnuhtrtJM6X1nQwoeJfczbRIDKC31xsYxEbY0Kf10SwVkRuACJFpJuI/AtYEMC46qyDR/P5auM+fnZmB1rENwx2OMYYU21eE8G9OOMV5wHv4HRH/ZsAxVSnvfb1FgqLlZEp1tW0MaZ+8DowTXdV/QPwh0AGU9dtP3CU177eynVntKP7KQnBDscYY2qE1zuCZ0RkvYg8WTIuQTh68tP1REUKv7+8e7BDMcaYGuMpEajqhTgjk2UAr4jIahH5Y0Ajq2PWph/my/V7uevCrrROiAl2OMYYU2M8t31U1T2q+gJwJ84zBeUOMFNf/WfhdmKiIrhpsD1AZoypX7w+UNZDRMaJyGqcwesX4AxGHxaO5Bbw0QrnuYEmcTbmgDGmfvFaWfwG8C5wqaqmBzCeOumTlekcKyiy7iSMMfWSp0SgqmcFOpC67OPUdBLjo+mT1DTYoRhjTI2rNBGIyHuqer1bJOT7JHHYjFCWkZVH6o5DXNmnTbBDMcaYgPB3R/Br9+eVgQ6krnpv6U7yi4q556KuwQ7FGGMCotLKYlXd7b69S1W3+76AuwIfXvB9sjKd/slN6dwyPtihGGNMQHhtPnpJOfMur8lA6qLNGdls2JPFsD42ML0xpv7yV0fwK5wr/84isspnUWPg20AGVhdMXbyDqEix+gFjTL3mr47gHeC/wP8DHvaZn6WqBwMWVR1QWFTMZ6t2c3aXRHuS2BhTr/krGlJV3QbcDWT5vBCR5oENLbi+XL+X9MO53DjYBqY3xtRvXu4IrgSW4TQfFZ9lCnQOUFxB98W6fTRu2IChPVoHOxRjjAmoShOBql7p/gyrYSkPHs1n2vI0xg5KJjJC/G9gjDEhzGtfQ+eISCP3/U0i8qyI1Nsyk9nr9wJwdT9rLWSMqf+8Nh+dAOSISF/gfmAz8La/jUTkMhHZKCKbROThcpb/TkTWicgqEZktInWiM5+Za/bQtkkMgzvV62oQY4wBvCeCQlVV4GrgRVV9CacJaYVEJBJ4Ced5g9OBsSJyepnVVgApblcVHwD/qErwgXA4p4D5P2QwrE8bRKxYyBhT/3lNBFki8gjwU+AzEYkA/PXHPAjYpKpbVDUfmIqTSEqp6lxVzXEnF1IHurZ+b+lOCoqUq/pasZAxJjx4TQSjcQau/4Wq7sE5YT/lZ5t2wE6f6TR3XkVuxXlm4QQicoeILBWRpRkZGR5DPjlzN+6jaVwUvds1Ceh+jDGmrvA6VOUeYDLQRESuBHJV9a2aCkJEbgJSqCC5qOqrqpqiqiktW7asqd2e4ODRfBZtPciNg5OtWMgYEza8thq6HlgMjAKuBxaJyEg/m+0C2vtMJ7nzyn72xcAfgOGqmuclnkB5f+lOioqVy3tZlxLGmPDhdYSyPwADVXUfgIi0BL7EqeCtyBKgm4h0wkkAY4AbfFcQkTOAV4DLSj47mOZu3EfzRtH0bJsQ7FCMMabWeK0jiChzoj7gb1tVLQTuAWYB64H3VHWtiIwXkeHuak8B8cD7IpIqIjOqFn7NycjKY/HWg9xkxULGmDDj9Y5gpojMAqa406OBz/1tpKqfl11PVR/zeX+xx/0H3P/W7aFY4QrradQYE2a8jln8oIhcB5zrznpVVacHLqza99/Ve+ic2IjTWlf6eIQxxtQ7/sYj6AY8DXQBVgMPqOoJFb6h7uDRfL7bcoBfDulsxULGmLDjr47gDeBTYAROD6T/CnhEQTB9xS6KipXh1reQMSYM+SsaaqyqE933G0VkeaADCobvNu+nc8tGdD/FWgsZY8KPv0QQ4zbxLCkvifWdVtWQTwyFRcV8uX4f1/Wv7KFnY4ypv/wlgt3Asz7Te3ymFbgoEEHVpnnfO11WDOpoPY0aY8KTv4FpLqytQILl6x/2ExsVycgBQe/vzhhjgsLrA2X11sq0Q/ROakKDyLD/KowxYSqsz375hcWsTT9Cv/ZNgx2KMcYETVgngg17jpBfWEzfpKbBDsUYY4LGa++j4o5V/Jg7nSwigwIbWuCt3HkIgL7tbewBY0z48npH8G/gLGCsO52FMwxlSJu1di8tGkXTrmlssEMxxpig8drp3GBV7S8iKwBUNVNEogMYV8CpKpv2ZXNq68bWrYQxJqx5vSMocAejVygdj6A4YFHVgrXpR9hzJJdrz7AHyYwx4c1rIngBmA60EpG/AN8Afw1YVLWg5EGyi3q0CnIkxhgTXF67oZ4sIsuAoTjdS1yjqusDGlmArdt9hI4t4kiMbxjsUIwxJqg8JQIRSQZygE9856nqjkAFFkj5hcUs2LSfc7u1DHYoxhgTdF4riz/DqR8QIAboBGwEegYoroBaveswmTkFXNHrlGCHYowxQee1aKi377SI9AfuCkhEtWD59kwAUqyjOWOMObkni93upwfXcCy1Zun2g3RoEUfLxlY/YIwxXusIfuczGQH0B9IDElGAqSrLtmcyxOoHjDEG8F5H4DuieyFOncG0mg8n8HYczGF/dj79OzQLdigmBBQUFJCWlkZubm6wQzHGk5iYGJKSkoiKivK8jd9E4D5I1lhVH6hOcHXFih2HAOifbInA+JeWlkbjxo3p2LGjPYFu6jxV5cCBA6SlpdGpUyfP21VaRyAiDVS1CDinugHWFdOWp9G8UTSnndLY/8om7OXm5tKiRQtLAiYkiAgtWrSo8h2svzuCxTj1AakiMgN4HzhaslBVP6xqoMG2cU8WnRMbERlh/9jGG0sCJpSczN+r1zqCGOAAzhjFJc8TKBBSiWDvkVz2ZeVxx5DOwQ7FGGPqDH/NR1u5LYbWAKvdn2vdn2sCHFuNW7HDeX7g9DYJQY7EGO/27t3LDTfcQOfOnRkwYABnnXUW06dPL3fd9PR0Ro4cWe6yCy64gKVLlwLwxhtv0Lt3b/r06UOvXr34+OOPAxb/tm3b6NWrV4XLn376abp3706/fv0YOHAgb731Fk888QSPPPLIceulpqbSo0ePcj9j5MiRbNmy5bh1RYSZM2dWGse4ceN4+umnK42luiZNmkS3bt3o1q0bkyZNKned1NRUzjzzTPr160dKSgqLFy8GYMOGDZx11lk0bNjwuDjz8/MZMmQIhYWF1Y4P/CeCSCDefTX2eV/yCinrd2cB0MMSgQkRqso111zDkCFD2LJlC8uWLWPq1KmkpaWdsG5hYSFt27blgw8+qPQz09LS+Mtf/sI333zDqlWrWLhwIX369Kl2rCdzUnr55Zf54osvWLx4MampqcyePRtVZezYsbz77rvHrTt16lTGjh17wmesXbuWoqIiOnf+8U5/ypQpnHvuuUyZMqXasVTHwYMHeeKJJ1i0aBGLFy/miSeeIDMz84T1HnroIR5//HFSU1MZP348Dz30EADNmzfnhRde4IEHjm+rEx0dzdChQ0/4jk6Wv6Kh3ao6vkb2VAekZR4jMT6aZo1CeigFEyRPfLKWdelHavQzT2+bwONXVdxTy5w5c4iOjubOO+8sndehQwfuvfdeAN58800+/PBDsrOzKSoqYtKkSVx55ZWsWbOGY8eO8fOf/5yVK1fSvXt3jh07BsC+ffto3Lgx8fHOtVx8fHzp+82bN3P33XeTkZFBXFwcEydOpHv37nzyySf8+c9/Jj8/nxYtWjB58mRat27NuHHj2Lx5M1u2bCE5OZnnnnuOO++8s/TqfMKECbRt25aioiJuv/12FixYQLt27fj444+JjY3lr3/9K1999RUJCc7FWUJCAjfffDMAzZo1Y9GiRQwe7Dy7+t577zFr1qwTvqPJkydz9dVXl06rKu+//z5ffPEF5513Hrm5ucTExPj9XVQWy8maNWsWl1xyCc2bO70YXHLJJcycOfOEhCYiHDni/G0dPnyYtm3bAtCqVStatWrFZ599dsJnX3PNNTzyyCPceOON1YoR/CeCelVLtu3AUbq0DLkbGRPG1q5dS//+/StdZ/ny5axatYrmzZuzbdu20vkTJkwgLi6O9evXs2rVqtLP6du3L61bt6ZTp04MHTqU6667jquuugqAO+64g5dffplu3bqxaNEi7rrrLubMmcO5557LwoULERFee+01/vGPf/DMM88AsG7dOr755htiY2MZPXo0559/PtOnT6eoqIjs7GwyMzP54YcfmDJlChMnTuT6669n2rRpDB8+nKysrOOu5H2NHTuWqVOnMnjwYBYuXEjz5s3p1q3bCet9++23x51YFyxYQKdOnejSpQsXXHABn332GSNGjKj0Ozxy5Eilsfh66qmnmDx58gnzhwwZwgsvvHDcvF27dtG+ffvS6aSkJHbt2nXCts899xyXXnopDzzwAMXFxSxYsMBvHL169WLJkiV+1/PCXyIYWiN7qSO2HzjK0O6tgx2GCVGVXbnXlrvvvptvvvmG6Ojo0pOA7xWnr/nz53PfffcB0KdPn9Lin8jISGbOnMmSJUuYPXs2v/3tb1m2bBkPPPAACxYsYNSoUaWfkZeXBzjFSaNHj2b37t3k5+cf10Z9+PDhxMY6w73OmTOntFw9MjKSJk2akJmZSadOnejXrx8AAwYMOC5hVWT06NGcffbZPPPMMxUWCwHs3r2bli1/7ClgypQpjBkzBoAxY8bw1ltvMWLEiApb01S1lc2DDz7Igw8+WKVt/JkwYQL//Oc/GTFiBO+99x633norX375ZaXbREZGEh0dTVZWFo0bV685fKV1BKp6sDofLiKXichGEdkkIg+Xs7yhiLzrLl8kIh2rs7/KZOUWsD87n46JjQK1C2NqXM+ePVm+fHnp9EsvvcTs2bPJyMgondeoUdX/pkWEQYMG8cgjjzB16lSmTZtGcXExTZs2JTU1tfS1fr0z7Mi9997LPffcw+rVq3nllVeOa6fuZf8NG/7Yr1dkZCSFhYUkJCQQHx9/XCWvr/bt29OpUyfmzZvHtGnTGD16dLnrxcbGlsZTVFTEtGnTGD9+PB07duTee+9l5syZZGVl0aJFixPK5w8ePEhiYqLfWHw99dRT9OvX74RXSdL11a5dO3bu3Fk6nZaWRrt2J46KOGnSJK677joARo0aVVpZ7E9eXp6nYi9/TqrTOS/cJ5JfAi4HTgfGisjpZVa7FchU1a7AP4G/ByqenQed8tGkZjZQvQkdF110Ebm5uUyYMKF0Xk5OjqdthwwZwjvvvAPAmjVrWLVqFeC0LPJNLqmpqXTo0IGEhAQ6derE+++/Dzhl7StXrgSccuuSE1hFLV8Ahg4dWhprUVERhw8frjTGRx55hLvvvru0fDw7O/u4ljpjx47lt7/9LZ07dyYpKancz+jRowebNm0CYPbs2fTp04edO3eybds2tm/fzogRI5g+fTrx8fG0adOGOXPmAE4SmDlzJueee66nWEo8+OCDxyXLklfZYiGASy+9lP/9739kZmaSmZnJ//73Py699NIT1mvbti3z5s0DnLuq8orAyjpw4ACJiYlV6kqiIgFLBMAgYJOqblHVfGAqcHWZda4GSv6qPgCGSoCe3vlstdNHXtum1c+extQWEeGjjz5i3rx5dOrUiUGDBnHzzTfz97/7v2b61a9+RXZ2Nj169OCxxx5jwIABgNN/0gMPPFDaTPLdd9/l+eefB5yK19dff52+ffvSs2fP0mal48aNY9SoUQwYMIDExMQK9/n8888zd+5cevfuzYABA1i3bp3fGC+88EIGDhxIr169OO+884iI+PG0NGrUKNauXVthsRDAsGHD+OqrrwCnWOjaa689bvmIESNKWw+99dZbPPnkk/Tr14+LLrqIxx9/nC5duniK5WQ0b96cP/3pTwwcOJCBAwfy2GOPlRbj3XbbbaXNeSdOnMj9999P3759efTRR3n11VcB2LNnD0lJSTz77LP8+c9/JikpqTRRzZ07l2HDhlUrvlKqGpAXMBJ4zWf6p8CLZdZZAyT5TG8GEsv5rDuApcDS5ORkPRlLth7QW99cokVFxSe1vQlP69atC3YIxo+cnBwdPHiwFhYWBjuUWnXttdfqxo0by11W3t8tsFQrOF8H8o6gxqjqq6qaoqopvpVCVZHSsTmv3ZxChHUtYUy9EhsbyxNPPFFua5z6Kj8/n2uuuYZTTz21Rj7PaxcTJ2MX0N5nOsmdV946aSLSAGiC05WFMcZ4Vl65e30WHR3Nz372sxr7vEDeESwBuolIJxGJBsYAM8qsMwMoeWJjJDDHvYUxps6wP0kTSk7m7zVgiUBVC4F7gFnAeuA9VV0rIuNFZLi72utACxHZBPwOOKGJqTHBFBMTw4EDBywZmJCg7ngEVW1SKqH2B56SkqIlNe3GBJqNUGZCTUUjlInIMlVNKW+bQNYRGBPyoqKiqjTSkzGhKCRaDRljjAkcSwTGGBPmLBEYY0yYC7nKYhHJALaf5OaJwP4aDCcU2DGHBzvm8FCdY+6gquU+kRtyiaA6RGRpRbXm9ZUdc3iwYw4PgTpmKxoyxpgwZ4nAGGPCXLglgleDHUAQ2DGHBzvm8BCQYw6rOgJjjDEnCrc7AmOMMWVYIjDGmDBXLxOBiFwmIhtFZJOInNCjqYg0FJF33eWLRKRjEMKsUR6O+Xcisk5EVonIbBHpEIw4a5K/Y/ZZb4SIqIiEfFNDL8csIte7v+u1IvJObcdY0zz8bSeLyFwRWeH+fV8RjDhrioi8ISL7RGRNBctFRF5wv49VItK/2jutaOiyUH0BkThDXnYGooGVwOll1rkLeNl9PwZ4N9hx18IxXwjEue9/FQ7H7K7XGJgPLARSgh13LfyeuwErgGbudKtgx10Lx/wq8Cv3/enAtmDHXc1jHgL0B9ZUsPwK4L+AAGcCi6q7z/p4RzAI2KSqW1Q1H5gKXF1mnauBSe77D4ChIhLKY1j6PWZVnauqOe7kQpwR40KZl98zwJPA34H60I+0l2O+HXhJVTMBVHVfLcdY07wcswIJ7vsmQHotxlfjVHU+cLCSVa4G3lLHQqCpiLSpzj7rYyJoB+z0mU5z55W7jjoD6BwGWtRKdIHh5Zh93YpzRRHK/B6ze8vcXlU/q83AAsjL7/lU4FQR+VZEForIZbUWXWB4OeZxwE0ikgZ8DtxbO6EFTVX/3/2y8QjCjIjcBKQA5wc7lkASkQjgWeCWIIdS2xrgFA9dgHPXN19EeqvqoWAGFWBjgTdV9RkROQt4W0R6qWpxsAMLFfXxjmAX0N5nOsmdV+46ItIA53byQK1EFxhejhkRuRj4AzBcVfNqKbZA8XfMjYFewFcisg2nLHVGiFcYe/k9pwEzVLVAVbcC3+MkhlDl5ZhvBd4DUNXvgBicztnqK0//71VRHxPBEqCbiHQSkWicyuAZZdaZAdzsvh8JzFG3FiZE+T1mETkDeAUnCYR6uTH4OWZVPayqiaraUVU74tSLDFfVUB7n1Mvf9kc4dwOISCJOUdGWWoyxpnk55h3AUAAR6YGTCDJqNcraNQP4mdt66EzgsKrurs4H1ruiIVUtFJF7gFk4LQ7eUNW1IjIeWKqqM4DXcW4fN+FUyowJXsTV5/GYnwLigffdevEdqjo8aEFXk8djrlc8HvMs4Ccisg4oAh5U1ZC92/V4zPcDE0XktzgVx7eE8oWdiEzBSeaJbr3H40AUgKq+jFMPcgWwCcgBfl7tfYbw92WMMaYG1MeiIWOMMVVgicAYY8KcJQJjjAlzlgiMMSbMWSIwxpgwZ4mgHhKRIhFJ9Xl1rGTd7BrY35sistXd13L36c6qfsZrInK6+/7RMssWVDdG93NKvpc1IvKJiDT1s36/k+nJUkTaiMin7vsLROSwu9/1IvL4SXze8JJeN0XkmpLvyZ0e7z4oWC3u73Ckn3W+qsoDee6xf+pnnfZuz6ElvaX+uoL1yu1xU0RaishMrzGZ8lkiqJ+OqWo/n9e2Wtjng6raD3gY58G1KlHV21R1nTv5aJllZ1c/PODH76UXzvMjd/tZvx9Oe+2q+h0w0Wf6a/e7ScHpE6dK3Qar6gxV/Zs7eQ1OD5slyx5T1S9PIsa6ohC4X1VPx3n6+27fROfjcpwnpLsBdwATAFQ1A9gtIufUUrz1kiWCMCAi8eKMQbBcRFaLyAm9dLpXsfN9rpjPc+f/RES+c7d9X0Ti/exuPtDV3fZ37metEZHfuPMaichnIrLSnT/anf+ViKSIyN+AWDeOye6ybPfnVBEZ5hPzmyIyUkQiReQpEVniXi3+0sPX8h1uR10iMsg9xhUiskBETnOfYh0PjHZjGe3G/oaILHbXLa+3U4ARwAlXqap6FFgGdHXvNha68U4XkWZuLPfJj+NGTHXn3SIiL4rI2cBw4Ck3pi4+38FlIvK+z3dTejVe1d+hiDzmfpdrRORVkeN65v2pz9/IIHd9r9/LCVR1t6oud99nAespvwO1ynrc/Ai40es+TTkC0Z+2vYL7wnmiNNV9Tcd5gjzBXZaI80RiycOE2e7P+4E/uO8jcfrqScQ5sTdy5/8eeKyc/b0JjHTfjwIWAQOA1UAjnCea1wJn4JwkJ/ps28T9+RXueAElMfmsUxLjtcAk9300Tg+MsThXiH905zcElgKdyokz2+f43gcuc6cTgAbu+4uBae77W4AXfbb/K3CT+74pTj8+jcrsoxOwzGf6AuBT930LYBvQE1gFnO/OHw88575PBxqW7KNsHL7fte+0+zve4fO7mgDcdJK/w+Y+898GrvL5HU103w/B7S+/ou+lzLGnAK/5+bvt6B5DQjnLPgXO9Zme7fP30g5YHez/u1B+1bsuJgzgFoGUTIhIFPBXERkCFOP847QG9vhsswR4w133I1VNFZHzcYohvnUvCqNxrqTL85SI/BGnj5dbcfp+ma7OVTAi8iFwHs6V8jMi8neck8TXVTiu/wLPi0hD4DJgvqoeE5GfAH18yrib4BQhbC2zfayIpLrHvx74wmf9SSLSDaeLgqgK9v8TYLiIPOBOxwDJ7meVaMOJ/dycJyIrcL77v+F0DNdUVee5yyfhJCZwEsRkEfkI50rXE3W6YpgJXCUiHwDDgIdwepn1+jsscaGIPATEAc1xkvgn7rIp7v7mi0iCOPUsFX0vvvEtBW6raIfuXco04DeqesTTQf9oH9C2itsYH5YIwsONQEtggKoWiNMbZ4zvCu4/9hCcE8ibIvIskAl8oapjPezjQVX9oGRCRIaWt5Kqfu+WkV8B/FlEZqvqeC8Hoaq5IvIVcCkwGmeQEnBGarpXVWf5+YhjqtpPROJw+q65G3gBZ/Cauap6rTgV619VsL0AI1R1Y2X7oMx3i1NHcGXph4g0qWT7YThX21cBfxCR3pWsW9ZU4B6c+o+lqprlFut4/R0iIjHAv3GutneKyDiOP56yfdIoFXwvItLa4z6jcJLAZFX9sILVKutxMwbnezcnyeoIwkMTYJ+bBC4EThivWJwxjPeq6kTgNZyh8hYC54hISZl/IxE51eM+vwauEZE4EWmEU6zztYi0BXJU9T84HeGVV3Fa4J4cyvMuTidbJXcX4JzUf1WyjYic6u6zXOqM1HYfcL/82A15yUnlFp9Vs3CKyErMAu4tKTMXp0fXsr7HKeKokKoeBjLFrYcBfgrME2cMhfaqOhenCKcJTrGar7Ix+ZqH833ezo9Jsqq/w5KT/n73Kr1sS6KSOp1zcXq9PIy376Vc7javA+tV9dlKVq2sx81TgXLH9zXeWCIID5OBFBFZDfwM2FDOOhcAK90ijNHA8+q0yLgFmCIiq3CKFLp72aE6FYBvAotx6gxeU9UVQG9gsVtE8zjw53I2fxVYJW5lcRn/wynu+FKdoQvBSVzrgOXiDPj9Cn7udt1YVuEMavIP4P+5x+673VzgdLdydDTOnUOUG9tad7rs5x4FNpeceCtxM05x2iqc1knjceou/uP+nlYAL+iJA8pMBR50K2W7lNl3EU5Z+uXuT6r6O3T3NxHnxDoLp8jQV677Pb2MUwQIHr4XcRoCvFbOLs/BSYQXyY/Nna9wt7lTRO501/scpzvtTW58d/l8xoVAfRmFLiis91FjapiIXItTDPfHYMcSDkRkPnC1uuM0m6qzOgJjapiqTheRUB4DO2SISEvgWUsC1WN3BMYYE+asjsAYY8KcJQJjjAlzlgiMMSbMWSIwxpgwZ4nAGGPC3P8HrdSCXONLo9sAAAAASUVORK5CYII=\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sensitity (PPV): \t 0.21414665801427643\n",
      "Specificity (TNR):   0.9645913043478261\n",
      "Precision: \t\t\t 0.49302788844621515\n",
      "False Positive Rate: 0.035408695652173916\n",
      "NPV: \t\t\t\t 0.8841702534672405\n"
     ]
    }
   ],
   "source": [
    "# Plot confusion matrix\n",
    "plot_confusion_matrix(cv_score_model, validation_data, validation_labels)\n",
    "plt.show()\n",
    "plot_roc_curve(cv_score_model, validation_data, validation_labels)\n",
    "plt.show()\n",
    "# Print all scores\n",
    "print_multiple_scores(validation_labels, pred_cv_score_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Pickle model\n",
    "pickle.dump(cv_score_model, open(\"RF_cv_score_model.p\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}