9-b_neural_networks_custom.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import math\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV\n",
    "from sklearn.metrics import classification_report, confusion_matrix, roc_auc_score, auc, f1_score, accuracy_score, precision_score, recall_score, roc_curve, precision_recall_curve \n",
    "from imblearn.over_sampling import SMOTE\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "data = pd.read_csv('data/model_data.csv')\n",
    "\n",
    "X_train = pd.read_csv('data/X_train_transformed.csv')\n",
    "X_test = pd.read_csv('data/X_test_transformed.csv')\n",
    "y_train = pd.read_csv('data/y_train_transformed.csv')\n",
    "y_test = pd.read_csv('data/y_test_transformed.csv')\n",
    "\n",
    "#this step is necessary because we are using the already splot data inside the pycaret setup, and indexes must not overlap on the train and test data sets\n",
    "\n",
    "# #combine balanced train data back into a single df\n",
    "# X_train['subscribed'] = y_train\n",
    "# train_data = X_train\n",
    "\n",
    "# #combine test data back into a single df\n",
    "# X_test['subscribed'] = y_test\n",
    "# test_data = X_test.set_index( np.arange(len(X_train), len(X_train)+len(X_test)))\n",
    "\n",
    "# X=train_data.drop(labels=['subscribed'], axis=1)\n",
    "# y=train_data['subscribed']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/64\n",
      "3/3 [==============================] - 1s 23ms/step - loss: 1.6841 - accuracy: 0.4998\n",
      "Epoch 2/64\n",
      "3/3 [==============================] - 0s 20ms/step - loss: 0.8581 - accuracy: 0.4627\n",
      "Epoch 3/64\n",
      "3/3 [==============================] - 0s 18ms/step - loss: 0.8797 - accuracy: 0.4996\n",
      "Epoch 4/64\n",
      "3/3 [==============================] - 0s 16ms/step - loss: 0.9248 - accuracy: 0.5000\n",
      "Epoch 5/64\n",
      "3/3 [==============================] - 0s 32ms/step - loss: 0.7644 - accuracy: 0.5092\n",
      "Epoch 6/64\n",
      "3/3 [==============================] - 0s 22ms/step - loss: 0.6897 - accuracy: 0.5481\n",
      "Epoch 7/64\n",
      "3/3 [==============================] - 0s 22ms/step - loss: 0.6969 - accuracy: 0.5340\n",
      "Epoch 8/64\n",
      "3/3 [==============================] - 0s 20ms/step - loss: 0.6640 - accuracy: 0.5858\n",
      "Epoch 9/64\n",
      "3/3 [==============================] - 0s 21ms/step - loss: 0.6339 - accuracy: 0.6660\n",
      "Epoch 10/64\n",
      "3/3 [==============================] - 0s 19ms/step - loss: 0.6303 - accuracy: 0.6672\n",
      "Epoch 11/64\n",
      "3/3 [==============================] - 0s 18ms/step - loss: 0.6084 - accuracy: 0.7018\n",
      "Epoch 12/64\n",
      "3/3 [==============================] - 0s 21ms/step - loss: 0.5948 - accuracy: 0.7014\n",
      "Epoch 13/64\n",
      "3/3 [==============================] - 0s 23ms/step - loss: 0.5863 - accuracy: 0.7019\n",
      "Epoch 14/64\n",
      "3/3 [==============================] - 0s 18ms/step - loss: 0.5715 - accuracy: 0.7294\n",
      "Epoch 15/64\n",
      "3/3 [==============================] - 0s 35ms/step - loss: 0.5646 - accuracy: 0.7390\n",
      "Epoch 16/64\n",
      "3/3 [==============================] - 0s 20ms/step - loss: 0.5548 - accuracy: 0.7387\n",
      "Epoch 17/64\n",
      "3/3 [==============================] - 0s 30ms/step - loss: 0.5489 - accuracy: 0.7347\n",
      "Epoch 18/64\n",
      "3/3 [==============================] - 0s 17ms/step - loss: 0.5426 - accuracy: 0.7393\n",
      "Epoch 19/64\n",
      "3/3 [==============================] - 0s 26ms/step - loss: 0.5378 - accuracy: 0.7435\n",
      "Epoch 20/64\n",
      "3/3 [==============================] - 0s 20ms/step - loss: 0.5328 - accuracy: 0.7434\n",
      "Epoch 21/64\n",
      "3/3 [==============================] - 0s 25ms/step - loss: 0.5291 - accuracy: 0.7429\n",
      "Epoch 22/64\n",
      "3/3 [==============================] - 0s 17ms/step - loss: 0.5251 - accuracy: 0.7445\n",
      "Epoch 23/64\n",
      "3/3 [==============================] - 0s 29ms/step - loss: 0.5220 - accuracy: 0.7459\n",
      "Epoch 24/64\n",
      "3/3 [==============================] - 0s 21ms/step - loss: 0.5187 - accuracy: 0.7457\n",
      "Epoch 25/64\n",
      "3/3 [==============================] - 0s 27ms/step - loss: 0.5158 - accuracy: 0.7480\n",
      "Epoch 26/64\n",
      "3/3 [==============================] - 0s 21ms/step - loss: 0.5128 - accuracy: 0.7497\n",
      "Epoch 27/64\n",
      "3/3 [==============================] - 0s 20ms/step - loss: 0.5102 - accuracy: 0.7503\n",
      "Epoch 28/64\n",
      "3/3 [==============================] - 0s 28ms/step - loss: 0.5076 - accuracy: 0.7496\n",
      "Epoch 29/64\n",
      "3/3 [==============================] - 0s 18ms/step - loss: 0.5050 - accuracy: 0.7515\n",
      "Epoch 30/64\n",
      "3/3 [==============================] - 0s 18ms/step - loss: 0.5027 - accuracy: 0.7527\n",
      "Epoch 31/64\n",
      "3/3 [==============================] - 0s 18ms/step - loss: 0.5004 - accuracy: 0.7525\n",
      "Epoch 32/64\n",
      "3/3 [==============================] - 0s 19ms/step - loss: 0.4978 - accuracy: 0.7533\n",
      "Epoch 33/64\n",
      "3/3 [==============================] - 0s 18ms/step - loss: 0.4954 - accuracy: 0.7540\n",
      "Epoch 34/64\n",
      "3/3 [==============================] - 0s 21ms/step - loss: 0.4931 - accuracy: 0.7558\n",
      "Epoch 35/64\n",
      "3/3 [==============================] - 0s 35ms/step - loss: 0.4912 - accuracy: 0.7582\n",
      "Epoch 36/64\n",
      "3/3 [==============================] - 0s 37ms/step - loss: 0.4892 - accuracy: 0.7598\n",
      "Epoch 37/64\n",
      "3/3 [==============================] - 0s 17ms/step - loss: 0.4875 - accuracy: 0.7608\n",
      "Epoch 38/64\n",
      "3/3 [==============================] - 0s 18ms/step - loss: 0.4857 - accuracy: 0.7613\n",
      "Epoch 39/64\n",
      "3/3 [==============================] - 0s 17ms/step - loss: 0.4840 - accuracy: 0.7635\n",
      "Epoch 40/64\n",
      "3/3 [==============================] - 0s 17ms/step - loss: 0.4824 - accuracy: 0.7646\n",
      "Epoch 41/64\n",
      "3/3 [==============================] - 0s 18ms/step - loss: 0.4810 - accuracy: 0.7654\n",
      "Epoch 42/64\n",
      "3/3 [==============================] - 0s 24ms/step - loss: 0.4795 - accuracy: 0.7658\n",
      "Epoch 43/64\n",
      "3/3 [==============================] - 0s 20ms/step - loss: 0.4782 - accuracy: 0.7669\n",
      "Epoch 44/64\n",
      "3/3 [==============================] - 0s 18ms/step - loss: 0.4769 - accuracy: 0.7683\n",
      "Epoch 45/64\n",
      "3/3 [==============================] - 0s 17ms/step - loss: 0.4759 - accuracy: 0.7695\n",
      "Epoch 46/64\n",
      "3/3 [==============================] - 0s 18ms/step - loss: 0.4748 - accuracy: 0.7712\n",
      "Epoch 47/64\n",
      "3/3 [==============================] - 0s 19ms/step - loss: 0.4740 - accuracy: 0.7714\n",
      "Epoch 48/64\n",
      "3/3 [==============================] - 0s 16ms/step - loss: 0.4732 - accuracy: 0.7721\n",
      "Epoch 49/64\n",
      "3/3 [==============================] - 0s 16ms/step - loss: 0.4724 - accuracy: 0.7721\n",
      "Epoch 50/64\n",
      "3/3 [==============================] - 0s 23ms/step - loss: 0.4718 - accuracy: 0.7710\n",
      "Epoch 51/64\n",
      "3/3 [==============================] - 0s 16ms/step - loss: 0.4710 - accuracy: 0.7715\n",
      "Epoch 52/64\n",
      "3/3 [==============================] - 0s 16ms/step - loss: 0.4706 - accuracy: 0.7722\n",
      "Epoch 53/64\n",
      "3/3 [==============================] - 0s 18ms/step - loss: 0.4700 - accuracy: 0.7710\n",
      "Epoch 54/64\n",
      "3/3 [==============================] - 0s 20ms/step - loss: 0.4696 - accuracy: 0.7720\n",
      "Epoch 55/64\n",
      "3/3 [==============================] - 0s 16ms/step - loss: 0.4691 - accuracy: 0.7716\n",
      "Epoch 56/64\n",
      "3/3 [==============================] - 0s 16ms/step - loss: 0.4686 - accuracy: 0.7726\n",
      "Epoch 57/64\n",
      "3/3 [==============================] - 0s 18ms/step - loss: 0.4683 - accuracy: 0.7724\n",
      "Epoch 58/64\n",
      "3/3 [==============================] - 0s 15ms/step - loss: 0.4678 - accuracy: 0.7722\n",
      "Epoch 59/64\n",
      "3/3 [==============================] - 0s 17ms/step - loss: 0.4674 - accuracy: 0.7749\n",
      "Epoch 60/64\n",
      "3/3 [==============================] - 0s 16ms/step - loss: 0.4671 - accuracy: 0.7721\n",
      "Epoch 61/64\n",
      "3/3 [==============================] - 0s 32ms/step - loss: 0.4666 - accuracy: 0.7749\n",
      "Epoch 62/64\n",
      "3/3 [==============================] - 0s 24ms/step - loss: 0.4664 - accuracy: 0.7736\n",
      "Epoch 63/64\n",
      "3/3 [==============================] - 0s 15ms/step - loss: 0.4659 - accuracy: 0.7763\n",
      "Epoch 64/64\n",
      "3/3 [==============================] - 0s 15ms/step - loss: 0.4659 - accuracy: 0.7730\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2085689ef10>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "from scikeras.wrappers import KerasClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "#Define the model\n",
    "def nn_model():\n",
    "    model = Sequential()\n",
    "    model.add(Dense(53, input_shape=(53,), activation='relu')) #Input layer\n",
    "    model.add(Dense(34, activation='relu'))\n",
    "    model.add(Dense(1, activation='sigmoid')) #Output layer\n",
    "\n",
    "    #Compile the model\n",
    "    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    return model\n",
    "\n",
    "#Declare parameteres\n",
    "\n",
    "# param_grid = {\n",
    "#     'batch_size': [16,32,64],\n",
    "#     'epochs': [5,10,15]\n",
    "# }\n",
    "\n",
    "# grid_search = GridSearchCV(\n",
    "#     estimator=KerasClassifier(build_fn=nn_model),\n",
    "#     param_grid=param_grid,\n",
    "#     cv=10\n",
    "# )\n",
    "\n",
    "#result = grid_search.fit(X_train,y_train)\n",
    "#Optimize parameteres\n",
    "\n",
    "#Fit the model\n",
    "model = nn_model()\n",
    "model.fit(X_train, y_train,batch_size = 17064, epochs = 64)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#grid_search.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "97/97 [==============================] - 0s 1ms/step - loss: 0.5104 - accuracy: 0.7636\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.5104081630706787, 0.7635966539382935]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# evaluate the model\n",
    "model.evaluate(X_test, y_test,batch_size=128)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "387/387 [==============================] - 1s 2ms/step\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>job_blue-collar</th>\n",
       "      <th>job_services</th>\n",
       "      <th>job_entrepreneur</th>\n",
       "      <th>job_admin.</th>\n",
       "      <th>job_management</th>\n",
       "      <th>job_technician</th>\n",
       "      <th>job_housemaid</th>\n",
       "      <th>job_unemployed</th>\n",
       "      <th>job_retired</th>\n",
       "      <th>...</th>\n",
       "      <th>poutcome_nonexistent</th>\n",
       "      <th>poutcome_failure</th>\n",
       "      <th>poutcome_success</th>\n",
       "      <th>emp.var.rate</th>\n",
       "      <th>cons.price.idx</th>\n",
       "      <th>cons.conf.idx</th>\n",
       "      <th>nr.employed</th>\n",
       "      <th>subscribed</th>\n",
       "      <th>prediction_score</th>\n",
       "      <th>prediction_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>34.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.584009</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>62.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.799088</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>26.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.290937</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>41.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.184112</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>52.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.177547</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12351</th>\n",
       "      <td>32.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.694001</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12352</th>\n",
       "      <td>59.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.691918</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12353</th>\n",
       "      <td>47.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.100229</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12354</th>\n",
       "      <td>45.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.556163</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12355</th>\n",
       "      <td>42.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.356218</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12356 rows × 56 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        age  job_blue-collar  job_services  job_entrepreneur  job_admin.  \\\n",
       "0      34.0              0.0           0.0               0.0         0.0   \n",
       "1      62.0              0.0           0.0               0.0         0.0   \n",
       "2      26.0              0.0           1.0               0.0         0.0   \n",
       "3      41.0              1.0           0.0               0.0         0.0   \n",
       "4      52.0              0.0           1.0               0.0         0.0   \n",
       "...     ...              ...           ...               ...         ...   \n",
       "12351  32.0              0.0           0.0               0.0         1.0   \n",
       "12352  59.0              0.0           0.0               0.0         0.0   \n",
       "12353  47.0              1.0           0.0               0.0         0.0   \n",
       "12354  45.0              0.0           0.0               0.0         0.0   \n",
       "12355  42.0              0.0           0.0               1.0         0.0   \n",
       "\n",
       "       job_management  job_technician  job_housemaid  job_unemployed  \\\n",
       "0                 1.0             0.0            0.0             0.0   \n",
       "1                 0.0             0.0            0.0             0.0   \n",
       "2                 0.0             0.0            0.0             0.0   \n",
       "3                 0.0             0.0            0.0             0.0   \n",
       "4                 0.0             0.0            0.0             0.0   \n",
       "...               ...             ...            ...             ...   \n",
       "12351             0.0             0.0            0.0             0.0   \n",
       "12352             0.0             0.0            0.0             0.0   \n",
       "12353             0.0             0.0            0.0             0.0   \n",
       "12354             0.0             1.0            0.0             0.0   \n",
       "12355             0.0             0.0            0.0             0.0   \n",
       "\n",
       "       job_retired  ...  poutcome_nonexistent  poutcome_failure  \\\n",
       "0              0.0  ...                   1.0               0.0   \n",
       "1              0.0  ...                   1.0               0.0   \n",
       "2              0.0  ...                   1.0               0.0   \n",
       "3              0.0  ...                   1.0               0.0   \n",
       "4              0.0  ...                   1.0               0.0   \n",
       "...            ...  ...                   ...               ...   \n",
       "12351          0.0  ...                   1.0               0.0   \n",
       "12352          0.0  ...                   1.0               0.0   \n",
       "12353          0.0  ...                   1.0               0.0   \n",
       "12354          0.0  ...                   1.0               0.0   \n",
       "12355          0.0  ...                   1.0               0.0   \n",
       "\n",
       "       poutcome_success  emp.var.rate  cons.price.idx  cons.conf.idx  \\\n",
       "0                   0.0           1.0             2.0            4.0   \n",
       "1                   0.0           1.0             1.0            0.0   \n",
       "2                   0.0           3.0             3.0            2.0   \n",
       "3                   0.0           3.0             4.0            1.0   \n",
       "4                   0.0           3.0             2.0            3.0   \n",
       "...                 ...           ...             ...            ...   \n",
       "12351               0.0           1.0             0.0            4.0   \n",
       "12352               0.0           1.0             1.0            0.0   \n",
       "12353               0.0           2.0             2.0            1.0   \n",
       "12354               0.0           1.0             1.0            1.0   \n",
       "12355               0.0           1.0             1.0            1.0   \n",
       "\n",
       "       nr.employed  subscribed  prediction_score  prediction_label  \n",
       "0              0.0           0          0.584009                 1  \n",
       "1              1.0           0          0.799088                 1  \n",
       "2              2.0           0          0.290937                 0  \n",
       "3              3.0           0          0.184112                 0  \n",
       "4              3.0           0          0.177547                 0  \n",
       "...            ...         ...               ...               ...  \n",
       "12351          0.0           0          0.694001                 1  \n",
       "12352          1.0           1          0.691918                 1  \n",
       "12353          3.0           0          0.100229                 0  \n",
       "12354          1.0           0          0.556163                 1  \n",
       "12355          1.0           0          0.356218                 0  \n",
       "\n",
       "[12356 rows x 56 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predicts = model.predict(X_test)\n",
    "predicts\n",
    "\n",
    "evaluate_data = X_test.copy()\n",
    "evaluate_data['subscribed'] = y_test\n",
    "evaluate_data['prediction_score'] = predicts\n",
    "evaluate_data['prediction_label'] = np.where(evaluate_data.prediction_score > 0.5, 1, 0)\n",
    "evaluate_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#save the nn model\n",
    "model.save('classifier_nn')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reloaded_model = tf.keras.models.load_model('classifier_nn')\n",
    "\n",
    "predictions = reloaded_model.predict(X_train)\n",
    "\n",
    "print(\n",
    "    \"This particular client had a %.1f percent probability \"\n",
    "    \"of subscribing to a term deposit, as evaluated by our model.\" % (150 * predictions[0][0],)\n",
    ")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#SHAP\n",
    "import shap\n",
    "\n",
    "background = X_train.iloc[np.random.choice(X_train.shape[0], 100, replace=False)]\n",
    "\n",
    "e = shap.DeepExplainer((model.layers[0].input, model.layers[-1].output), background)\n",
    "shap_values = e.shap_values(X_test[1:5])\n",
    "\n",
    "# plot the feature attributions\n",
    "shap.beeswarm(shap_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAT4AAAEWCAYAAAD/x/trAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmhUlEQVR4nO3dedwWVf3/8dcbUEBBFlFEXLBEEDdcAvdcEa1EyzUXMosytcX8lVqJufQtWywzLFMK1NxyARUXRMk0EVARBTVRM9mUTVQQ5cbP7485N17SvVwD93Jd9/V++pjHNXPmzJkzoB/PzJk5RxGBmVkladXcFTAza2oOfGZWcRz4zKziOPCZWcVx4DOziuPAZ2YVx4GvhZHUXtLdkpZKum0dyjlJ0oMNWbfmIOk+SUObux5WWhz4momkL0uaKuk9SfPSf6D7NkDRxwDdgY0j4ti1LSQiboyIQQ1Qn0+QdICkkHTnGum7pPSJRZZzkaQb6ssXEYdHxKi1rK61UA58zUDSOcBvgZ+RBamtgBHAkAYofmvg3xFR1QBlNZYFwF6SNi5IGwr8u6FOoIz//baaRYSXJlyATsB7wLF15GlLFhjnpuW3QNu07wBgNvB94C1gHnBa2vdT4ENgZTrH6cBFwA0FZfcCAmiTtr8CvAq8C7wGnFSQ/ljBcXsDU4Cl6Xfvgn0TgUuAx1M5DwLdarm26vr/ETgzpbUG5gAXAhML8v4OeAN4B3gK2C+lD17jOp8tqMdlqR7vA9umtK+l/VcDtxeU/wtgAqDm/vfCS9Mu/j9i09sLaAfcWUeeHwF7Av2BXYABwI8L9m9GFkB7kgW3P0jqEhHDyVqRt0REh4i4rq6KSNoQuBI4PCI6kgW3aTXk6wrcm/JuDPwGuHeNFtuXgdOATYH1gXPrOjcwGjg1rR8GPE8W5AtNIfsz6Ar8DbhNUruIuH+N69yl4JhTgGFAR+D1Ncr7PrCTpK9I2o/sz25oRPi7zQrjwNf0NgYWRt23oicBF0fEWxGxgKwld0rB/pVp/8qIGEfW6umzlvX5CNhRUvuImBcRM2rI8zng5Yi4PiKqIuIm4EXgCwV5/hIR/46I94FbyQJWrSLiX0BXSX3IAuDoGvLcEBGL0jl/TdYSru86/xoRM9IxK9cobznZn+NvgBuAsyNidj3lWQvkwNf0FgHdJLWpI8/mfLK18npKW13GGoFzOdAhb0UiYhlwPPBNYJ6keyX1LaI+1XXqWbA9fy3qcz1wFnAgNbSAJZ0r6YXUQ/02WSu3Wz1lvlHXzoh4kuzWXmQB2iqQA1/TewL4ADiqjjxzyTopqm3F/94GFmsZsEHB9maFOyPigYg4FOhB1or7cxH1qa7TnLWsU7XrgW8B41JrbLV0K/oD4DigS0R0Jnu+qOqq11Jmnbetks4kaznOTeVbBXLga2IRsZTsIf4fJB0laQNJ60k6XNLlKdtNwI8lbSKpW8pf76sbtZgG7C9pK0mdgPOrd0jqLmlIetb3Adkt80c1lDEO2C69gtNG0vFAP+CetawTABHxGvBZsmeaa+oIVJH1ALeRdCGwUcH+N4FeeXpuJW0HXAqcTHbL+wNJ/deu9lbOHPiaQXpedQ5Zh8UCstuzs4C7UpZLganAdOA54OmUtjbnGg/cksp6ik8Gq1apHnOBxWRB6IwaylgEfJ6sc2ARWUvp8xGxcG3qtEbZj0VETa3ZB4D7yV5xeR1YwSdvY6tfzl4k6en6zpMeLdwA/CIino2Il4ELgOsltV2Xa7DyI3domVmlcYvPzCqOA5+ZVRwHPjOrOA58ZlZx6nqJtsmpTfvQ+h2buxqWQ99P96w/k5WMuXP+y9uLF6n+nLVrvdHWEVXvF5U33l/wQEQMXpfzNYbSCnzrd6Rtn+OauxqWw+g7LmvuKlgOpx55wDqXEVUraNv3hKLyrnjm9/V9adMsSirwmVkZEKB1ajQ2Owc+M8uvzIc6dOAzs/zc4jOzyiJo1bq5K7FOHPjMLB/hW10zqzTyra6ZVSC3+Mys4rjFZ2aVRW7xmVmFEe7VNbNKU/4tvvKuvZk1j1YqbqmHpO9JmiHpeUk3SWonaRtJT0qaJekWSeunvG3T9qy0v1dBOeen9JckHVZv9dfl2s2sAlW/x1fMUlcxUk/g28AeEbEj0Bo4AfgFcEVEbAssIZv4nfS7JKVfkfIhqV86bgdgMDBCUp334g58ZpafVNxSvzZA+zQZ1AbAPOAg4O9p/yg+nop1SNom7T9YklL6zRHxQZq5bxYwoK6TOvCZWU7pk7ViFugmaWrBMqy6lIiYA/wK+C9ZwFtKNhPg2xFRlbLN5uOJ63uSZtpL+5cCGxem13BMjdy5YWb5Fd+5sTAi9qixCKkLWWttG+BtsilDm2TQUrf4zCyfYm9z67/VPQR4LSIWRMRK4A5gH6BzuvUF2AKYk9bnAFtmVVAboBPZPM+r02s4pkYOfGaWXwN0bpDd4u4paYP0rO5gYCbwCHBMyjMUGJPWx6Zt0v6HI5sYfCxwQur13QboDUyu68S+1TWz/Brgk7WIeFLS34GngSrgGeAa4F7gZkmXprTr0iHXAddLmgUsJuvJJSJmSLqVLGhWAWdGxKq6zu3AZ2Y5NdwLzBExHBi+RvKr1NArGxErgGNrKecyoOgJYBz4zCwff7JmZpWn/D9Zc+Azs/w8LJWZVRy3+Mys4rjFZ2YVRX7GZ2YVSK0c+MysggiQb3XNrKIoLWXMgc/McpJbfGZWeRz4zKzitHLnhplVFD/jM7NKIz/jM7NK5MBnZhXHgc/MKo4Dn5lVFoFalXfgK+8+aTNrctWdG8UsdZYj9ZE0rWB5R9J3JXWVNF7Sy+m3S8ovSVdKmiVpuqTdCsoamvK/LGlo7WfNOPCZWW4NEfgi4qWI6B8R/YHdgeXAncB5wISI6A1MSNsAh5PNoNYbGAZcnerSlWzejoFkc3UMrw6WtXHgM7P8VORSvIOBVyLidbJJxkel9FHAUWl9CDA6MpPI5t/tARwGjI+IxRGxBBhPPROT+xmfmeWjXJ0b3SRNLdi+JiKuqSHfCcBNab17RMxL6/OB7mm9J/BGwTGzU1pt6bVy4DOz3HIEvoURsUc9Za0PHAmcv+a+iAhJkb+GdfOtrpnlIkSrVq2KWop0OPB0RLyZtt9Mt7Ck37dS+hxgy4LjtkhptaXXyoHPzPJr2Gd8J/LxbS7AWKC6Z3YoMKYg/dTUu7snsDTdEj8ADJLUJXVqDEpptfKtrpnlk+8ZX91FSRsChwLfKEj+OXCrpNOB14HjUvo44AhgFlkP8GkAEbFY0iXAlJTv4ohYXNd5HfjW0tknHchXjt6biGDGrLkMG34Dv//RCey3+7YsfW8FAMMuvJ7p/57DRh3aMfLSoWzZowttWrfmt6MncP3YSQCc9IWBnPe1wwD4+bUPcOPdTzbbNbVk8+fO5qJzv8nihQtA4ugThnLiaWdw9W8u5dHx41CrVnTdeBOG/3IEm3TvwfXXXMl9Y24FYNWqVfxn1ks8OPUVOnXuwr/+8RC/vvg8PvpoFUOOO5WvnPG9Zr66ptdQgS8ilgEbr5G2iKyXd828AZxZSzkjgZHFnteBby1svkknvnXiZ9n1S5ex4oOV3PCLr3LsYbsDcMFv7+LOh6Z9Iv83jtufF1+dzzHf/RPdunTg2Tt/ws3jptBhg7b8aNjh7HPS5UQE//rbD7l34nTefvf9Zriqlq1NmzZ894JL6btjf5a99y6nHnkAA/c9kFO+/m3OOOfHANz81z9y7ZWXc/5lV3DKsG9zyrBvA/DohPu4aeQIOnXuwqpVq7h8+LlcNfouum+2OUOPOpD9DzmcT/Xu25yX1+TK/ZM1P+NbS21at6Z92/Vo3boV7dutz7wFS2vNG0CHDdsCsGH7tixZupyqVR9x6N7bM2HSiyx5Zzlvv/s+Eya9yKB9+jXRFVSWbptuRt8d+wOwYYeO9Np2OxbMn0eHjhutzvP+8uU1/gf94NjbGfSFYwCY8exTbLn1p9hiq16st/76HPr5L/GP8eOa5BpKiVqpqKVUOfCthbkLlvLb0RP4932X8Nr4y3jnvSxoAVx05heYfMv5XP79L7L+elmD+o83/4O+22zGqw9extTbLuDcX/6diGDzTToz+80lq8ud89bbbL5J5+a4pIoyd/brvDTjOXbon7XSR/zqEj63zw7cP/Y2vvG9Cz6Rd8X7y3ni0Yc4aPCRACyYP4/uPT5+Rax7j81Z8OY8KkmxX22UcquwUQOfpMGSXkrf1p1X/xHloXPH9nz+gJ3Y/vPD+dSgH7Fh+/U54YjPcOHvx7LL0Zew78m/pEunDfn+aYcAcOje2zP9pdl8atCPGHjC/3HFecfSccN2zXwVlWn5svf44bdO5Zyf/Gx1a+9b5/6Eex+fweAjj+XW0Z98t/bRCfez8+4D6dS5zi+gKo4DXy0ktQb+QPaOTj/gREkt4j7uoIF9+c/cRSxc8h5VVR9x18PPsucu2zB/4TsAfLiyitFjJrHHDr0AOOXIPRnz8LMAvPrGQv4zZxF9enVn7oK32aL7x/9B9dy0M3MXvN3Ul1Mxqlau5IffOpXBRx67ugVX6PAhx/LwA3d/Im38PbdzWLrNBdhksx68Oe/jV8TenDeXTbr3aLxKlygHvtoNAGZFxKsR8SFwM9m3dmXvjfmLGbDTNrRvtx4ABw7ow0uvvclm3T5+XnTkgTsz85W5Kf8SDhjQB4BNu3Zku17deW3OQsb/6wUO2asvnTu2p3PH9hyyV1/G/+uFpr+gChARXHLeWfT69Hac9LWzVqf/97VXVq//46Fx9PpU79Xb772zlKeffJzPHnrE6rR+O+/Gf//zCnPe+A8rP/yQ8ffczv6HHN40F1FKGv5b3SbVmL26NX0/N3DNTJKGkY20AOt1aMTqNJwpz7/OnQ89wxN/+yFVqz7i2Rdnc93tjzPmqjPo1qUjEkx/aTZnX3YzAD//8/1c89OTmXLrBUjwo9+NYdHbywD4vz/fz2M3/ACAn11zP0veWd5s19WSPTt1EuPuvIVt+/Tjy5/bF4Azz72QMbdez+uvzaKVxGY9t+T8S69YfcwjD97DwP0Oov0GG65Oa9OmDT+46Jd8e+iXWPXRKo489mQ+vd32TX49za2UW3PFUPZqTCMULB0DDI6Ir6XtU4CBEXFWbce02mDTaNvnuNp2Wwn65x2XNXcVLIdTjzyAmc89s05Rq91mvWOLk68sKu8rvz7iqfq+1W0Ojdniy/39nJmVg9J+fleMxnzGNwXoLWmbNPrCCWTf2plZmZOKW0pVo7X4IqJK0llkHwu3BkZGxIzGOp+ZNZ1yb/E16idrETGO7MNiM2spSrw1Vwx/q2tmuQhoVcKfoxXDgc/McnPgM7PK4ltdM6s0wp0bZlZxyv89Pgc+M8utzOOex+Mzs5yUdW4Us9RblNRZ0t8lvSjpBUl7Seoqabykl9Nvl5RXkq5Mw9xNl7RbQTlDU/6XJQ2t/YwZBz4zy6X6GV8DDUv1O+D+iOgL7AK8AJwHTIiI3sCEtA3ZEHe90zIMuJqsLl2B4WSDoAwAhlcHy9o48JlZbg3xyZqkTsD+wHUAEfFhRLxNNnzdqJRtFHBUWh8CjI7MJKBzmnf3MGB8RCyOiCXAeGBwXed24DOz3HK0+LpJmlqwDCsoZhtgAfAXSc9IujZNN9k9zZcLMB/ontZrGuquZx3ptXLnhpnllqNzY2Edw1K1AXYDzo6IJyX9jo9va4FsSklJDT52nlt8ZpaPGuwZ32xgdkRUTyb9d7JA+Ga6hSX9vpX21zbUXe4h8Bz4zCwXUVyPbn29uhExH3hDUp+UdDAwk2z4uuqe2aHAmLQ+Fjg19e7uCSxNt8QPAIMkdUmdGoNSWq18q2tmuTXge3xnAzemMTtfBU4ja5DdKul04HWgelj2ccARwCxgecpLRCyWdAnZGKAAF0fE4rpO6sBnZrk11JcbETENqOkZ4ME15A3gzFrKGQmMLPa8Dnxmlo8HKTCzSuNBCsysIjnwmVnF8UCkZlZZ/IzPzCqNPB6fmVWiMo97Dnxmll+rMo98DnxmlovUgjs3Ckc3rUlEPN3w1TGzclDmca/OFt+v69gXwEENXBczKxMttnMjIg5syoqYWfko87hX/7BUkjaQ9GNJ16Tt3pI+3/hVM7NSJNIrLUX8U6qKGY/vL8CHwN5pew5waaPVyMxKXisVt5SqYgLfpyPicmAlQEQshxIO5WbWuNQwA5E2p2JeZ/lQUnuyDg0kfRr4oFFrZWYlS1TGe3zDgfuBLSXdCOwDfKUxK2Vmpa3M4179gS8ixkt6GtiTLNh/JyIWNnrNzKxklfvrLMVONvRZsqGgDwT2a7zqmFmpK3Yy8WJio6T/SHpO0jRJU1NaV0njJb2cfrukdEm6UtIsSdMLP7KQNDTlf1nS0NrOV62Y11lGAN8EngOeB74h6Q/1X5KZtVStpaKWIh0YEf0L5t89D5gQEb2BCXw81+7hQO+0DAOuhixQkj2SGwgMAIZXB8vaFPOM7yBg+zTRB5JGATOKvSIza3ka+VZ3CHBAWh8FTAR+mNJHp1g0SVLnNO/uAcD46pnVJI0HBgM31XaCYm51ZwFbFWxvmdLMrAJlvbpFv8fXTdLUgmXYGsUF8KCkpwr2dU/z5QLMB7qn9Z7AGwXHzk5ptaXXqq5BCu5OleoIvCBpctoeCEyuq1Aza8GUayDShQW3sDXZNyLmSNoUGC/pxcKdERGSYm2rWpu6bnV/1dAnM7OWoaHudCNiTvp9S9KdZM/o3pTUIyLmpVvZt1L2OWR3nNW2SGlz+PjWuDp9Yl3nrWuQgn/kvAYzqxAN8YxP0oZAq4h4N60PAi4GxgJDgZ+n3zHpkLHAWZJuJrvzXJqC4wPAzwo6NAYB59d17no7NyTtCfwe2B5YH2gNLIuIjfJdppm1BAJaN8znaN2BO1MQbQP8LSLulzQFuFXS6cDrwHEp/zjgCLI+huXAaQARsVjSJcCUlO/i6o6O2hTTq3sVcAJwG7AHcCqwXfHXZmYtTUOEvYh4FdilhvRFZO8Nr5kewJm1lDUSGFnsuYt6gTkiZgGtI2JVRPyFrKvYzCqQlH2rW8xSqopp8S2XtD4wTdLlwDyK/+LDzFqgEo5pRSkmgJ2S8p0FLCPrVfliY1bKzEqb0ist9S2lqphBCl5PqyuAnwJIugU4vhHrZWYlrIRjWlHWdnrJvRq0FmZWNiQ1VK9us/G8umaWWynfxhZjbebVFbBeY1Rm1+234vEnr2qMoq2RrFi5qrmrYDm0Xa91g5RT7r2bazuv7ot17DOzFky04Baf59U1s9qU+SM+P+Mzs3ykBvtkrdk48JlZbmUe9xz4zCy/Mn/EV9ScG5J0sqQL0/ZWkgY0ftXMrBRVz6tbzt/qFtMrPYLsheUT0/a7gCcbMqtgrYpcSlUxt7oDI2I3Sc8ARMSSNGiBmVWoEm7MFaWYwLdSUmuy+TaQtAnwUaPWysxKVqV8snYlcCewqaTLgGOAHzdqrcyspJV53CtqdJYbJT1FNiKqgKMi4oVGr5mZlaTqzo1yVkyv7lZk49vfTTbZx7KUZmYVSipuKa4stZb0jKR70vY2kp6UNEvSLdV9CpLapu1ZaX+vgjLOT+kvSTqsvnMW0/FyL3BP+p0AvArcV9wlmVmLU+Rk4jluh78DFN5F/gK4IiK2BZYAp6f004ElKf2KlA9J/cjmBdqBbFqMEalfolb1Br6I2Ckidk6/vcnmvXyi6EsysxZHRf5TbznSFsDngGvTtoCDgL+nLKOAo9L6kLRN2n9wyj8EuDkiPoiI18hmYavzXePcX25ExNOSBuY9zsxaBgFtin9Jr5ukqQXb10TENQXbvwV+AHRM2xsDb0dEVdqeDfRM6z2BNwAiokrS0pS/JzCpoMzCY2pUzLy65xRstgJ2A+bWd5yZtVw5hqVaGBF71FLG54G3IuIpSQc0UNWKUkyLr2PBehXZs77bG6c6Zlbqsl7dBilqH+BISUcA7YCNgN8BnSW1Sa2+LYA5Kf8cssnOZktqA3QCFhWkVys8pkZ1Br70gLBjRJyb+5LMrGXK0WNbl4g4HzgfILX4zo2IkyTdRva+8M3AUGBMOmRs2n4i7X84IkLSWOBvkn4DbA70BibXde66hp5vk+6j91mHazOzFqiR3+P7IXCzpEuBZ4DrUvp1wPWSZgGLyXpyiYgZkm4FZpLdlZ4ZEXXOiVBXi28y2fO8aSmi3kY2ry7pZHes1SWZWVkT0LqBRyCIiInAxLT+KjX0ykbECuDYWo6/DLis2PMV84yvHdl99EFk3+sq/TrwmVUk0aqIV1VKWV2Bb9PUo/s8Hwe8atGotTKzkpVNNtTctVg3dQW+1kAHqDG0O/CZVap8X2WUpLoC37yIuLjJamJmZaPcBymoK/CV95WZWaNo6be6BzdZLcysrLTYgUgjYnFTVsTMyoMo7fk0iuHpJc0sH+X6VrckOfCZWW7lHfYc+Mwsp5Yw9LwDn5nlVt5hz4HPzHITrVpqr66ZWU3cq2tmFcm9umZWcco77DnwmVlefo/PzCqNgNYOfGZWaco77JV/54yZNQOpuKXuMtRO0mRJz0qaIemnKX0bSU9KmiXpFknrp/S2aXtW2t+roKzzU/pLkg6rr/4OfGaWS/Y6i4pa6vEBcFBE7AL0BwZL2hP4BXBFRGwLLAFOT/lPB5ak9CtSPiT1I5t4aAdgMDAizRBZKwc+M8utIVp8kXkvba6XliCb3+fvKX0UcFRaH5K2SfsPVtbLMgS4OSI+iIjXgFnUMFlRIQc+M8tJRf8DdJM0tWAZ9omSpNaSpgFvAeOBV4C302TiALOBnmm9J/AGQNq/FNi4ML2GY2rkzg0zyyVnr+7CiNijtp1p/tv+kjoDdwJ917mCRXCLz8zyKfI2N88bLxHxNvAIsBfQWVJ1o2wLYE5anwNsCZD2dyKb+nZ1eg3H1MiBz8xya6Be3U1SSw9J7YFDgRfIAuAxKdtQYExaH5u2SfsfjohI6SekXt9tgN7A5LrO7VtdM8tNDfMmXw9gVOqBbQXcGhH3SJoJ3CzpUuAZ4LqU/zrgekmzgMVkPblExAxJtwIzgSrgzHQLXSu3+BpAn217sUf/nRi4e3/2GZg9zrj04ov41NY9Gbh7fwbu3p/77xsHwKJFizjskAPp1rkD3/32Wc1Y68o24ve/Za/dd2avPXbh9KEnsWLFCr417Kvssv227Ddwd/YbuDvPPTsNgMcenchWm3VdnX75zy5p3so3s2wg0uKWukTE9IjYNSJ2jogdq6ezjYhXI2JARGwbEcdGxAcpfUXa3jbtf7WgrMsi4tMR0Sci7qvvGtziayD3P/QI3bp1+0Ta2d/5Ht8759xPpLVr144LL7qEmTOeZ8aM55uyipbMnTOHP424iklPP0f79u057eQTuOO2WwC4+Ge/YMjRX/qfY/bae19uuWNsU1e1ZJX7CMxu8TWxDTfckH323Zd27do1d1UqWlVVFSvef5+qqiqWL1/OZj16NHeVykqO11lKkgNfA5DEFw4fxN4Ddue6P1+zOv2PI67iM7vuzDe+9lWWLFnSjDW0Qpv37MnZ3z2HnfpsQ99PbcFGnTpx0CGDALj0op+wz4BdueAH5/DBBx+sPmbK5EnsO3A3jhnyOV6YOaO5ql4SGupWtzk1WuCTNFLSW5Ja/P3chImP8cSUp7nrnvv409V/4LF/PsrXv3EGM196hSefmsZmPXpw3v/7fnNX05K3lyxh3D1jmTZzFi+88gbLly3jlptu5MKfXsbkaTN4+J+TWLJkCb/79eUA7Nx/N6a/+CqPPfk0w844k5OP/99b4cqS6wXmktSYLb6/kn031+L17Jm9JL7pppty5FFHM2XKZLp3707r1q1p1aoVXz3960ydWmfvujWhiY9MYOutt6HbJpuw3nrr8YUhRzN50hNs1qMHkmjbti0nnTKUp6ZOAWCjjTaiQ4cOAAwafAQrV65k0cKFzXkJzasR3uNrao0W+CLiUbIu5xZt2bJlvPvuu6vXHxr/IDvssCPz5s1bnWfMXXfSb4cdm6uKtoYtttiSqVOeZPny5UQE/5j4MH369mV++juLCO69eyzb77ADAG/On0/2uhg8NWUyH330EV033rjZ6l8KVORSqpq9Vzd9uzcMYMuttmrm2uT31ptvcvwxRwNQtaqK40/4MoMOG8xXh57C9GenIYmte/Xi9yP+tPqYPtv24t133uHDDz/k7rF3cc+4B9m+X7/muoSKs8eAgRx51Bc5YO/P0LpNG3bepT9Dv/p1jj3qcyxcuJCIYKedd+E3V44AYMydt/OXa/9E6zZtaN+uHdeNvrHsRyBeFy1hIFJV/5+sUQrPxsu6JyKKau7svvse8fiTUxutPtbwVqys8z1RKzEH7jOQZ56euk5Ra/uddo2/3PVIUXn32rbLU3V9q9tcmr3FZ2blp5Q7LorhwGdmuZX5nW6jvs5yE/AE0EfSbEmn13eMmZUHd27UIiJObKyyzayZlXJUK4Jvdc0sF6n8v9V14DOz3Mo77DnwmdnaKPPI58BnZjmV9ne4xXDgM7PcyvwRnwOfmeUjyj/weTw+M8utIYalkrSlpEckzZQ0Q9J3UnpXSeMlvZx+u6R0SbpS0ixJ0yXtVlDW0JT/ZUlDaztnNQc+M8utgYalqgK+HxH9gD2BMyX1A84DJkREb2BC2gY4nGwGtd5kA5tcndVFXYHhwEBgADC8OljWxoHPzHJriC83ImJeRDyd1t8lm1qyJzAEGJWyjQKOSutDgNGRmUQ2/24P4DBgfEQsjoglwHjqGQvUz/jMLJ9G+B4tjeS0K/Ak0D0iqge0nA90T+s9gTcKDpud0mpLr5UDn5nlluN1lm6SCseauyYirinMIKkDcDvw3Yh4p3Csw4gISQ0+dp4Dn5nlUj3ZUJEW1jUen6T1yILejRFxR0p+U1KPiJiXbmXfSulzgC0LDt8ipc0BDlgjfWJdlfIzPjPLrwEe8ilr2l0HvBARvynYNRao7pkdCowpSD819e7uCSxNt8QPAIMkdUmdGoNSWq3c4jOz3Broy419gFOA5yRNS2kXAD8Hbk1D2b0OHJf2jQOOAGYBy4HTACJisaRLgCkp38URUed8Pw58ZpZbQ7zAHBGPUXu78OAa8gdwZi1ljQRGFntuBz4zy63MP9xw4DOztVDmkc+Bz8xy8UCkZlaRyjvsOfCZ2doo88jnwGdmOXkgUjOrQGX+iM+Bz8zyaQkDkTrwmVluvtU1s4rjFp+ZVZwyj3sOfGaWU3HDypc0Bz4zWwvlHfkc+Mwsl5wDkZYkBz4zy823umZWcfw6i5lVnvKOew58ZpZfmcc9TzZkZvlIxS/1l6WRkt6S9HxBWldJ4yW9nH67pHRJulLSLEnTJe1WcMzQlP9lSUNrOlchBz4zy01SUUsR/goMXiPtPGBCRPQGJqRtgMOB3mkZBlyd6tIVGA4MBAYAw6uDZW0c+MwstwaYXRKAiHgUWHNGtCHAqLQ+CjiqIH10ZCYBndO8u4cB4yNicUQsAcbzv8H0E/yMz8xya+TXWbqn+XIB5gPd03pP4I2CfLNTWm3ptXLgM7Occg1E2k3S1ILtayLimmIPjoiQFLmqVwQHPjPLJed4fAsjYo+cp3hTUo+ImJduZd9K6XOALQvybZHS5gAHrJE+sa4T+BmfmeXWUL26tRgLVPfMDgXGFKSfmnp39wSWplviB4BBkrqkTo1BKa1WbvGZWW4N9eWGpJvIWmvdJM0m6539OXCrpNOB14HjUvZxwBHALGA5cBpARCyWdAkwJeW7OCLW7DD5BAc+M8unAYeliogTa9l1cA15AzizlnJGAiOLPa8Dn5nlUuyrKqXMgc/M8ivzyOfAZ2a5eXQWM6s4HojUzCqPA5+ZVRrf6ppZRcn55UZJUvZqTGmQtIDshcWWphuwsLkrYbm01L+zrSNik3UpQNL9ZH8+xVgYEXWOlNIcSirwtVSSpq7F94rWjPx31rL5W10zqzgOfGZWcRz4mkbR449ZyfDfWQvmZ3xmVnHc4jOziuPAZ2YVx4GvEUkaLOmlNA/oefUfYc2tpnlereVx4GskkloDfyCbC7QfcKKkfs1bKyvCX6lnakIrfw58jWcAMCsiXo2ID4GbyeYFtRJWyzyv1sI48DWe3HN9mlnTcOAzs4rjwNd4apsD1MyamQNf45kC9Ja0jaT1gRPI5gU1s2bmwNdIIqIKOItsYuMXgFsjYkbz1srqk+Z5fQLoI2l2mtvVWhh/smZmFcctPjOrOA58ZlZxHPjMrOI48JlZxXHgM7OK48BXJiStkjRN0vOSbpO0wTqU9VdJx6T1a+saPEHSAZL2Xotz/EfS/8zEVVt6LWV8RdJVDXFes0IOfOXj/YjoHxE7Ah8C3yzcKWmt5kiOiK9FxMw6shwA5A58ZqXMga88/RPYNrXG/ilpLDBTUmtJv5Q0RdJ0Sd8AUOaqNDbgQ8Cm1QVJmihpj7Q+WNLTkp6VNEFSL7IA+73U2txP0iaSbk/nmCJpn3TsxpIelDRD0rVk804XRdIASU9IekbSvyT1Kdi9Zarjy5KGFxxzsqTJqV5/SsOAmRVlrVoJ1nxSy+5w4P6UtBuwY0S8JmkYsDQiPiOpLfC4pAeBXYE+ZOMCdgdmAiPXKHcT4M/A/qmsrhGxWNIfgfci4lcp39+AKyLiMUlbkX2Zsj0wHHgsIi6W9DkgzxcPLwL7RUSVpEOAnwFfSvsGADsCy4Epku4FlgHHA/tExEpJI4CTgNE5zmkVzIGvfLSXNC2t/xO4juwWdHJEvJbSBwE7Vz+/AzoBvYH9gZsiYhUwV9LDNZS/J/BodVkRUduYdIcA/aTVDbqNJHVI5/hiOvZeSUtyXFsnYJSk3kAA6xXsGx8RiwAk3QHsC1QBu5MFQoD2wFs5zmcVzoGvfLwfEf0LE9J/9MsKk4CzI+KBNfId0YD1aAXsGREraqjL2roEeCQijk631xML9q35TWWQXeeoiDh/XU5qlcvP+FqWB4AzJK0HIGk7SRsCjwLHp2eAPYADazh2ErC/pG3SsV1T+rtAx4J8DwJnV29I6p9WHwW+nNIOB7rkqHcnPh6y6ytr7DtUUldJ7YGjgMeBCcAxkjatrqukrXOczyqcA1/Lci3Z87un02Q5fyJr1d8JvJz2jSYbfeQTImIBMAy4Q9KzwC1p193A0dWdG8C3gT1S58lMPu5d/ilZ4JxBdsv73zrqOT2NfDJb0m+Ay4H/k/QM/3sXMhm4HZgO3B4RU1Mv9I+BByVNB8YDPYr8MzLz6CxmVnnc4jOziuPAZ2YVx4HPzCqOA5+ZVRwHPjOrOA58ZlZxHPjMrOL8f5EpBu7odFTeAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix,ConfusionMatrixDisplay\n",
    "import matplotlib.pyplot as plt\n",
    "y_true = evaluate_data['subscribed']\n",
    "y_pred = evaluate_data['prediction_label']\n",
    "cm = confusion_matrix(y_true, y_pred)\n",
    "\n",
    "# disp.plot()\n",
    "\n",
    "# plt.show()\n",
    "\n",
    "# cm\n",
    "\n",
    "# Plot the confusion matrix\n",
    "plt.imshow(cm, interpolation='nearest', cmap=plt.cm.Blues)\n",
    "plt.title('Confusion Matrix')\n",
    "plt.colorbar()\n",
    "tick_marks = np.arange(len(np.unique(y_true)))\n",
    "plt.xticks(tick_marks, np.unique(y_true))\n",
    "plt.yticks(tick_marks, np.unique(y_true))\n",
    "plt.xlabel('Predicted Label')\n",
    "plt.ylabel('True Label')\n",
    "\n",
    "# Fill the confusion matrix cells with values\n",
    "thresh = cm.max() / 2.0\n",
    "for i, j in np.ndindex(cm.shape):\n",
    "    plt.text(j, i, format(cm[i, j], 'd'), horizontalalignment=\"center\",\n",
    "             color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "# Show the plot\n",
    "plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.78      0.85     10950\n",
      "           1       0.27      0.61      0.37      1406\n",
      "\n",
      "    accuracy                           0.76     12356\n",
      "   macro avg       0.60      0.70      0.61     12356\n",
      "weighted avg       0.86      0.76      0.80     12356\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.58400875],\n",
       "       [0.79908824],\n",
       "       [0.29093656],\n",
       "       ...,\n",
       "       [0.10022936],\n",
       "       [0.5561628 ],\n",
       "       [0.35621846]], dtype=float32)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predicts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Threshold=0.560513\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "from numpy import argmax\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(y_test,predicts)\n",
    "\n",
    "# get the best threshold\n",
    "J = tpr - fpr\n",
    "ix = argmax(J)\n",
    "best_thresh = thresholds[ix]\n",
    "print('Best Threshold=%f' % (best_thresh))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA0yklEQVR4nO3dd3xV9fnA8c+TBAiEMJIAskLYQ0DAAArVKjhAEWcduBBb+qviqFbBUUEcdVtXW1Goo9ZRQEVRkVYRRDCAIyxRdsJehhESMp7fH+cGL8lNcjPOnc/79cqLe9Y9z4Fwn3vO9/t9vqKqGGOMiV4xwQ7AGGNMcFkiMMaYKGeJwBhjopwlAmOMiXKWCIwxJsrFBTuAqkpJSdG0tLRgh2GMMWFl2bJlu1W1ma9tYZcI0tLSWLp0abDDMMaYsCIim8rbZo+GjDEmylkiMMaYKGeJwBhjolzYtRH4UlBQQHZ2Nnl5ecEOJSTFx8fTpk0b6tSpE+xQjDEhKCISQXZ2NomJiaSlpSEiwQ4npKgqe/bsITs7m/bt2wc7HGNMCHLt0ZCITBORnSKyopztIiLPishaEckUkX7VPVdeXh7JycmWBHwQEZKTk+1uyRhTLjfvCF4BngdeK2f7cKCz52cg8HfPn9ViSaB89ndjTBjLyoCNC2DnD7BlKXQfCWfeX6uncC0RqOp8EUmrYJfzgdfUqYO9WESaiEhLVd3mVkzGGBOysjJg4V9h7edQmAt1G0LLvrDpSxRnugABZx+o1WQQzF5DrYEsr+Vsz7oyRGSsiCwVkaW7du0KSHBVJSLcfvvtR5efeOIJJk2a5PfxO3bsYMSIEZxwwgn06NGDc845B4B58+YxYsSIMvvPmjWLRx55BIBJkybxxBNPADB69GimT59egysxxgREVga8NQoebAmTGsPUM+GH2U4SADhyEDYtABTBkwRKrJ5Vq6GERWOxqk4BpgCkp6eH5Ew69erVY+bMmdx1112kpKRU+fj77ruPM888k1tuuQWAzMzMCvcfOXIkI0eOrFasxpggysqAN6+A3N2V7qr8kgC8X9O9dv/vB/OOYAvQ1mu5jWddWIqLi2Ps2LE8/fTTZbZt3LiRIUOG0Lt3b4YOHcrmzZvL7LNt2zbatGlzdLl3795l9lmyZAl9+/Zl3bp1vPLKK4wbN652L8IYU/tKvvm/NASeH+B8868kCajnB/3ltQA0agWDbw2fNgI/zALGichbOI3EObXVPnDZi4vKrBvRuyVXn5zG4SNFjP5nRpntl5zYht+kt2XvoSP84V/Ljtn29u9P9uu8N954I7179+bOO+88Zv1NN93Etddey7XXXsu0adO4+eabee+998oce9lll/H8889zxhlncN1119GqVauj27/66ituuukm3n//fVJTU1mwYIFfMRljAigrA965Fg5sBYmF+smQu9Pvw0sedwhwJK4hB1PPICk/CxJbwuBboO0AV8J2LRGIyJvAaUCKiGQDE4E6AKr6D+Aj4BxgLZALXOdWLIHSqFEjrrnmGp599lnq169/dP2iRYuYOXMmAFdffXWZRAFw9tlns379ej755BM+/vhj+vbty4oVTs/b1atXM3bsWD799NNjkoMxJgSUNPL+9F8oyv9lvRb5nQQUKCaGH0kl6+QHOOvskdQFktyI1wc3ew1dUcl2BW5049wVfYOvXze2wu1JCXX9vgPw5dZbb6Vfv35cd13V81pSUhKjRo1i1KhRjBgxgvnz55OcnEzLli3Jy8vj22+/tURgTLDNnQgZL0FRHhADxQU1ersjTbvw+0Yv8PmaXfRNbcJjfco+Fnab1RqqZUlJSVx66aVMnTr16LpBgwbx1ltvAfDGG29wyimnlDnus88+IzfX6S1w4MAB1q1bR2pqKgBNmjRh9uzZ3HXXXcybN8/9izDGOOZOhGf7Os/2J6c4vXsW/hUKDkFxUdWTQEpXjjb5pnTl/QtW0W/vgyxev5f7RvRg+v8NonOLxNq+ikqFRa+hcHP77bfz/PPPH11+7rnnuO6663j88cdp1qwZ//znP8scs2zZMsaNG0dcXBzFxcX89re/pX///kc/+Fu0aMGHH37I8OHDmTZtWqAuxZjIV6b/fiIkd4IdK2r4bV+gcVs4vA+apsGIp8o842+8Zid92jbhLxf1om1Sg5pcRY2I84QmfKSnp2vpiWlWr15N9+7dgxRReLC/I2O8vHYhrP/MnfeOrQsn3eCzZ09hUTFTv9xAQVEx44Z0Bpx6YIEY/S8iy1Q13dc2uyMwxkSuuRNh0d+g+AjE1oP4xpC7x2nIrQ0xdZzeQSmdfX7j97Zq637Gz8hk+ZYczu3d8mgCCIUSMJYIjDGRofQjntKK8uGQ/105y5BY0GKIiYVm3Sv94C+RX1jE85+t5e/z1tGkQR3+dmU/hvc8LiQSQAlLBMaY8HO0v37J0KNafMRdNxFSusC+9U6tn/a/grRTqt2Hf+PuXP7xxTpG9mnFn8/tQdOEurUXay2xRGCMCW1ZGfCvSyA/x3n+3qofZC2uvfeXWGiQBH2urLURu4fyC5m7agcX9G1N1+MS+d9tp5GaHLzG4MpYIjDGhJ65E2Hhc0CpZ/lFR2qQBGKhYTPnZVw9OK6XK6N1F/y0i7tmLmfLz4fp2boRnZonhnQSAEsExphQUDJISwuhXhM4tKMW3lQArbAXT23KyS3goY9W8c7SbDqkJPD22JPp1DzwYwKqwxJBLRERbrvtNp588knAKUN98OBBJk2axKRJk3jsscfYuHEjzZs3B6Bhw4YcPHgwmCEbEzwlDbsbv4K8fcduK6xmEohrAJ2GuFqTpzxFxcrF//iKDbsPccNpHbl5aGfi68QGNIaasERQSyorQ52SksKTTz7Jo48+GoTojAmS1y6EjfMhpi7EN4Ijh6Agt/rdNyUWel4Me9e5XojNH3sPHaFJ/TrExgh3nN2V1k3q07N146DFU13RmwhKpn+rQW8Ab95lqB966KEy28eMGcMrr7zC+PHjSUoKVCkpYwIkKwPe/T3s3YDPHjzFhXDQR5dOfwXo8Y6/VJWZ32xh8oerGD+sG6MGpnL28ccFO6xqi7xE8PEE2L684n3y9zvDx7UYJAZa9IR6jcrf/7heMPyRSk9dXhlqcB4FjRkzhmeeeYb77w+NX2ZjamTuRGemrKJCyCk7x0a11U2E+k2h8HCt9uSpLdn7crn73RXM/3EXJ7ZryoD24f/FLvISgT/ycpwkAM6feTkVJwI/lVeGusTNN99Mnz59+NOf/lTjcxkTNI93qaXG3NIEOpwO17zrwnvXjne/zebed1egwP0jj+fqk9oRExM6A8OqK/ISgR/f3MnKgFdHOl3RYuvCxS/X2nPGispQN2nShFGjRvHCCy/UyrmMCZiSXj0Fh6r/HvWbOncPhfkQEwf1EkLyG39FkhLqcWJaEg9f2JM2TUO7S2hVRF4i8EfbAXDtrFptIyjhXYZ6zJgxZbbfdttt9O/fn8LCwlo7pzGumDsRFj4LFFfv+LgGTjmGcipvhoOComJeWrCewiLl5qGd+XWXZpzaOSWkykPUhuhMBOD8Urr0i1m6DLW3lJQULrzwQp9zGxsTdN5TLVaFxECdBtD/t2H1Db8iK7bkMH5GJiu37ue8E1qFVJG42mZlqKOE/R2ZCs34HSx/p+rHxcXDwP+LmA9/gLyCIp7930+8OH89TRvU5cELjmdYz5bBDqvGrAy1Maas5wfAnp9+6ThRFXUaOo26Yfi4pzKb9uTy0oL1XNS3Nfee24PGDeoEOyTXWSIwJtpkZcDUYZSp41OZmDpOm1oI9+qprkP5hcxZuZ2L+rWh63GJfHb7aUGdMSzQIiYRBGqWn3AUbo//jEuODvpa7/8xEgM9L4GLX3IvriD74sdd3D1zOVtzDtO7TWM6NU+MqiQAEZII4uPj2bNnD8nJyZYMSlFV9uzZQ3x8fLBDMYHmXb65KuLi4doPIvKxj7d9h47wwOxVzPxmCx2bJfCf34dPkbjaFhGJoE2bNmRnZ7Nr165ghxKS4uPjadOmTbDDMIEyZQhsXVb14+o1hqumR3wCgF+KxG3ak8u40zsxbkinsCoSV9siIhHUqVOH9u3bBzsMYwLH+9t+fFNo3RcO58DWb6lyv/+UrjAuw5UwQ82eg/k0bVCX2BhhwrButG5an+NbhV+RuNoWEYnAmKjgPRG7t7x9sO6zqr9f3US4O7t2Ygtxqsp/lmXz4IerGD+8G1cObMdZYVwkrrZZIjAmFJV07Uzu7BRFXDmj5u8Z1yAsyzrUVNbeXO5+dzkLftrNgLQkTu6QHOyQQo4lAmNCQXk9enavcX5qosOQiOzy6Y+Z32Rz73srEOCBC3py5YDUiCgSV9ssERgTbNUd1Vsith6kDXbaCPath4TmUFwA3UdG1Td/X1Ia1mNA+yQeurAXrZuUrQhsHJYIjAmWmpZzDrHJWkJBQVExL36xjqJiuOWMzpzapRmndmkW7LBCniUCYwLttQthfRUad1O6eh4ZKTTrHraVPN22YksOd0zPZPW2/Zzfp5UNMq0CSwTGuOloHf/DTg3+4iL8Lu0gsTDoJvvGX4m8giL++t+feGnBepIS6vLi1SeG9bSRweBqIhCRYcAzQCzwsqo+Ump7KvAq0MSzzwRV/cjNmIxxXXmTuJTu9ulLbD3480534opQm/fmMvXL9VzSrw13n9M9KorE1TbXEoGIxAIvAGcC2cASEZmlqqu8drsXeEdV/y4iPYCPgDS3YjLGVc8PqH4Pn5g6cPKN9u3fTwfyCvhkxXZ+k96WLi0S+fxPp0XUjGGB5uYdwQBgraquBxCRt4DzAe9EoEDJZMGNgSrOhmFMkNV4CkeBwbdYAqiCz3/YyT3vLmf7/jz6pjahU/NESwI15GYiaA1keS1nAwNL7TMJ+FREbgISgDN8vZGIjAXGAqSmptZ6oMZUSXkjfCsTEwf1EqFeIziul5MArNHXb3sPHeGBD1fx7rdb6Ny8IdP/MChqi8TVtmA3Fl8BvKKqT4rIycDrItJT9diZMlR1CjAFnBnKghCniXZZGfDKeVCUV7XjJBaa97CePjVUVKxc8vev2Lw3l5uHdubG0ztSLy56i8TVNjcTwRagrddyG886b9cDwwBUdZGIxAMpgLWWmeDKyoDZt8H25dU7XmJhzCf24V9Duw7kk5zgFIm7+5zutG5an+4tG1V+oKkSNxPBEqCziLTHSQCXA6NK7bMZGAq8IiLdgXjAakmb4Khu+WZv1uWzVqgq7yzN4sHZqxk/rBtXndSOM3q0CHZYEcu1RKCqhSIyDpiD0zV0mqquFJHJwFJVnQXcDrwkIn/EaTgerTadlgmkmvT0ASAGEpKjrpCbmzbvyWXCzEy+WreHge2T+FWnlGCHFPFcbSPwjAn4qNS6+7xerwIGuxmDMcc4+shnBc53j2qq2xD6/9Y+/GvZ9GXZ/Pm9FcTGCA9d2JMr+luRuEAIdmOxMYFzfxJoFSds9xbBk7eHihaN6jGoYzIPXtiTlo2tSFygWCIwkWvG72DFjJp9+Ce0gDt+rL2YzDGOFBbz93nrKFblj2d24ZTOzTilsxWJCzRLBCYyPdAcivKreFAMDL3X+dZvvX1c933Wz9w5PZM1Ow5wUd/WViQuiCwRmMhSndLO9rw/oA4fKeKpuWuY+uUGmifG8/I16dYjKMgsEZjIMHciLPyr//s3ToU/VnOMgKmRrH25vPrVJi4fkMqE4d1oFG9F4oLNEoEJb1Xq/hkDTdPgohft0U+A7fcUibvUUyRu3h2n0cpmDAsZlghM+Knq1I7xTWHCRtfCMRX77Icd3D1zBTsP5NEvtSmdmje0JBBiLBGY8DB3Iix8Dr8ndSkRxRO3B9ueg/lM/nAV73+3la4tEvnH1SfSqXnDYIdlfLBEYEJPdat7lpAYGDPHHv8EUVGx8pt/LCJrXy5/PKMLfzitI3XjYoIdlimHJQITOqr6yMcXuwMIqp0H8khJqEdsjHDPud1p07QBXY+zUtGhzu9EICINVDXXzWBMlKppvZ8WvazMc5AVFytvLtnMXz76gfHDu3H1Se0Y2t26hIaLShOBiAwCXgYaAqkicgLwe1W9we3gTISrapdPb/bhHzI27j7EhJmZLF6/l0Edk/m1jQwOO/7cETwNnA3MAlDV70XkVFejMpHttQth/WdVO8bm9A1J7yzN4s/vraBubAyPXNSLy/q3tdHBYcivR0OqmlXqH7cGxVtM1KpSAhC4/lP7xh/iWjepz6ldmvHA+T05rnF8sMMx1eRPIsjyPB5SEakD3AKsdjcsEzGq8+3fGnxDVn5hEX/7fB2qym1ndWVwpxQG23wBYc+fRPB/wDM4k9FvAT4FrH3AlK+60zwOvtUe/YSwbzfvY/yMTH7ccZCL+7WxInERxJ9E0FVVr/ReISKDgYXuhGTCVnV7/6R0hXEZtR+PqRW5Rwp58tMfmbZwA8c1imfa6HSGdLMeQZHEn0TwHNDPj3UmGlXn0U+JVifC2GoeawJmy77DvL54E1cOTGX8sG4kWpG4iFNuIhCRk4FBQDMRuc1rUyOcOYhNtHskDfL2Ve0Ym9w9LOQcLuDj5du4fEAqnVsk8sUdp9mMYRGsojuCujhjB+IA76GB+4FL3AzKhIFJjau2f69L4eKX3InF1KpPV27n3vdWsOfQEdLTkujUvKElgQhXbiJQ1S+AL0TkFVXdFMCYTCiryiAwm+YxrOw+mM+kWSv5MHMb3Y5L5OVr061IXJTwp40gV0QeB44HjnYUVtUhrkVlQk9WBkwdRqVDSGy2r7BUVKxc8vev2PpzHn86qwu//3VH6sRakbho4U8ieAN4GxiB05X0WmCXm0GZEOJvTyCr+R+WduzPo1lDp0jcxPOOp03T+nRuYUXioo0/KT9ZVacCBar6haqOAexuIJLNnQiTkpx2AH+SwIhnLAmEmeJi5fXFmxj65Be88bXz5Pf0bs0tCUQpf+4ICjx/bhORc4GtQJJ7IZmgeroX5Gz2c2crAxGO1u86yISZy8nYsJdfdUrhtK7Ngx2SCTJ/EsGDItIYuB1n/EAj4FY3gzJBMHciLHwWKPZvfysDEZbeXrKZ+95fSb24GB67pDe/ObGNjQ42lScCVf3Q8zIHOB2Ojiw2keLBllDox1QTNgYg7LVp2oDTujpF4po3siJxxlHRgLJY4FKcGkOfqOoKERkB3A3UB/oGJkTjGn9HBcfEwTlPQvpo10MytSu/sIjn/rcWgD+dbUXijG8V3RFMBdoCGcCzIrIVSAcmqOp7AYjNuGlSU/x6DDQpx/VQjDuWbdrLndMzWbfrEJemW5E4U76KEkE60FtVi0UkHtgOdFTVPYEJzbjC3wFhNhgsbB3KL+TxOWt4ddFGWjWuz6tjBvDrLjZrmClfRYngiKoWA6hqnoisr2oSEJFhOCWsY4GXVfURH/tcCkwCFPheVUdV5RymCvxJAlYILuxt/fkw/87YzDUnteOOYd1oWM/vqclNlKroN6SbiGR6XgvQ0bMsgKpq74re2NPG8AJwJpANLBGRWaq6ymufzsBdwGBV3Sci1o/NDZNToLig4n0kBsbMsa6gYSont4DZy7cxaqBTJG7BnafTwhqDjZ8qSgTda/jeA4C1qroeQETeAs4HVnnt8zvgBVXdB6CqO2t4TlNi7kRY+Bx+zSpqdwFh7ZMV2/nz+yvYe+gIAzsk0bFZQ0sCpkoqKjpX00JzrYEsr+VsYGCpfboAiMhCnMdHk1T1k9JvJCJjgbEAqampNQwrQk0ZAluXVf04mxUsbO08kMekWSv5aPl2erRsxD9H96djMysSZ6ou2A8P44DOwGlAG2C+iPRS1Z+9d1LVKcAUgPT0dA1wjKHvgeZQlF/14ywJhK2iYuXSfyxia04ed5zdlbGndrAicaba3EwEW3C6n5Zo41nnLRv4WlULgA0i8iNOYljiYlyRo7qzg8XWhZNusCQQhrblHKZFYrxTJG7k8bRt2sBKRZsa8ysRiEh9IFVVqzIh7RKgs4i0x0kAlwOlewS9B1wB/FNEUnAeFa2vwjmiU1YGTD0Lp6OVnxqkwBVvWmNwmCouVl5btJHH5qxhwvBuXHNyGqdbjSBTSypNBCJyHvAEzoxl7UWkDzBZVUdWdJyqForIOGAOzvP/aaq6UkQmA0tVdZZn21kisgqnVfMOG6dQiYfbwJED/u0bEwfHX2Qzg4W5tTsPMmFGJks37ePULs0Y0s0SgKldolrxt0oRWYZTdnqeqvb1rFuuqr0CEF8Z6enpunTp0mCcOrj8nRcgpg7ct9v9eExAvJWxmftmraR+nVjuG9GDi/q1ttHBplpEZJmqpvva5lcZalXNKfXLZw22geTv/MDW+BtxUpMbcEb35tw/sifNEusFOxwTofxJBCtFZBQQ6xkAdjPwlbthGcCrLaASKV1hXIb78RjX5RUU8ez/fgLgzmHdGNQxhUEdrUiccZc/ieAm4B4gH/g3znP9B90MKur5Ww8orgHcu831cExgLN24lztnZLJ+1yEu79/WisSZgPEnEXRT1XtwkoFx2yNpkLev8v3sMVDEOJhfyOOf/MBrizfRukl9XhszgFOtSJwJIH8SwZMichwwHXhbVVe4HFP08qctoHEq/HG5+7GYgNmec5i3lmRx7clp3HF2VxKsSJwJsEqHIqrq6Tgzk+0CXhSR5SJyr+uRRRt/kkCHIZYEIsS+Q0d4fbFTxaVTc6dI3KSRx1sSMEHh12+dqm7HmZzmc+BO4D6snaB2+DMuwBqDI4aq8vGK7dz3/gp+zi1gUMdkOjZraNNGmqDyZ0BZd+Ay4GJgD/A2zkT2pqb8uQuwGcIixs79efz5/RXMWbmDXq0b89qYgVYkzoQEf+4IpuF8+J+tqltdjic6+NMrKL4pTNgYiGhMABQVK795cRHbc/K4a3g3rv9Ve+KsSJwJEZUmAlU9ORCBRI0HW0JhbsX7jHjGJoqPEFt/PsxxjZwicZPP70nbpvXpYHcBJsSUmwhE5B1VvVRElnPsSGK/ZigzPlQ2YbwVhosYRSVF4j5Zw13nOEXibN5gE6oquiO4xfPniEAEEvEeSaPCJGCzhEWMtTsPcOf0TL7Z/DOndW3G0O4tgh2SMRWqaIaykiGrN6jqeO9tIvIoML7sUaaMrAyYOowKp4y0wWER499fb2bSrJUk1Ivl6ctO4II+ViTOhD5/GovPpOyH/nAf60xplY0StsFhESctpQFnHd+CSSOPJ6WhFYkz4aGiNoI/ADcAHUQk02tTIrDQ7cDC3uQUKC4of7slgYiQV1DE0//9EUGYMNyKxJnwVNEdwb+Bj4G/ABO81h9Q1b2uRhXuXrvQkkAU+Hr9HibMXM6G3Ye4cmCqFYkzYauiRKCqulFEbiy9QUSSLBmUIyuj4nmEr59rvYLC3IG8Ah795Af+tXgzqUkN+PdvBzKok90FmPBV2R3BCGAZTvdR7686CnRwMa7wlJUBU88sf7uNEo4IO/bnM31ZNr/9VXtuO6sLDepafSAT3irqNTTC82f7wIUT5iwJRKy9h44wO3MrV5+cRqfmDVlw5xCbMcxEDH9qDQ0GvlPVQyJyFdAP+KuqbnY9unDySFr5266fG7AwTO1SVT7M3MakWSvZn1fA4E4pdGjW0JKAiSj+FDv5O5ArIifgFJtbB7zualThZu7E8ruJWptA2NqxP4/fvbaMm978ltZN6/PBTb+y8hAmIvnzcLNQVVVEzgeeV9WpInK924GFlYXP+F4/+FZLAmGqqFi51FMk7p5zunPd4DQrEmcilj+J4ICI3AVcDZwiIjFAHXfDCiNZGRxbiskjpo6NFg5D2ftyadm4PrExwgPn9yQ1qQFpKQnBDssYV/nzFecynInrx3gmqGkDPO5qVOHk1fN8r79vd2DjMDVSVKy8vGA9Zzz1Bf/yzBx2apdmlgRMVPCnDPV2EXkD6C8iI4AMVX3N/dDCwNyJUJhXdn2HIYGPxVTbmu0HuHNGJt9n/czQbs0563grEmeiiz+9hi7FuQOYhzOW4DkRuUNVp7scW2iraHKZa94NaCim+v61eBP3f7CSxPg6PHN5H0ae0MpGB5uo408bwT1Af1XdCSAizYD/AtGdCMpLAmKDi8JBSTmITs0bck6vltw3ogfJViTORCl/PrViSpKAxx78a1uIXBWNGRjzccDCMFV3+EgRT81dQ0yMcNfw7pzUIZmTOiQHOyxjgsqfRPCJiMwB3vQsXwZ85F5IIS4rw8YMhKlF6/YwYWYmm/bkcvVJ7axInDEe/jQW3yEiFwG/8qyaoqrR+xB82jDf623MQMjan1fAXz76gTczNtMuuQH//t1AKxVtjJeK5iPoDDwBdASWA39S1S2BCixkqY+ZxmzMQEjbuT+f977dwthTO/DHM7pQv25ssEMyJqRU9Kx/GvAhcDFOBdLnqvrmIjJMRNaIyFoRmVDBfheLiIpIelXPEVBTyukWamMGQs6eg/m8snADAJ2aN+TL8adz9zndLQkY40NFj4YSVfUlz+s1IvJNVd5YRGKBF3CmuswGlojILFVdVWq/ROAW4OuqvH/AZWXA1mXBjsJUQlWZ9f1WJs1aycH8Qk7t0owOzRpajyBjKlBRIogXkb78Mg9Bfe9lVa0sMQwA1qrqegAReQs4H1hVar8HgEeBO6oYe2BNPcv3+pSugY3DlGvrz4e5970VfPbDTvq0bcJjl/S2InHG+KGiRLANeMprebvXsgKVDZ9tDWR5LWcDA713EJF+QFtVnS0i5SYCERkLjAVITU2t5LQueO1CfNYTIgbGZQQ6GuNDYVExl09ZzK4D+fx5RA9GD0ojNsZ6BBnjj4ompjndzRN7itc9BYyubF9VnQJMAUhPT/f1ieyu8qaenFRON1ITMFl7c2nVpD5xsTE8fGEvUpMakJrcINhhGRNW3BwYtgVo67XcxrOuRCLQE5gnIhuBk4BZIddg/HAb3+vtkVBQFRYVM2X+Os546gteX7QRgF91TrEkYEw1uFkPYQnQWUTa4ySAy4FRJRtVNQc42plbRObhdFFd6mJMVXfkgO/19kgoaFZv28/4GZlkZudwZo8WDO/VMtghGRPWXEsEqlooIuOAOUAsME1VV4rIZGCpqs5y69y1Zu5E3+tt6smgeX3RRu7/YBWN69fh+VF9ObdXSxsdbEwN+VN9VIArgQ6qOllEUoHjVLXSr8Sq+hGlylGo6n3l7HuaXxEH0ndv+F5vI4gDrqQcRJcWiZx3Qiv+PKIHSQl1gx2WMRHBnzuCvwHFOL2EJgMHgBlAfxfjCg2HfAwUs7aBgMo9UsgTc34kLla4+5zuDOyQzEArEmdMrfKnsXigqt4I5AGo6j4gSr6K+eigZG0DAbNw7W7O/ut8pi3cwJHCYlQD32HMmGjgzx1BgWeUsMLR+QiKXY0qFDxvj3+CJedwAQ/PXs3bS7Non5LAO78/mQHtk4IdljERy59E8CzwLtBcRB4CLgHudTWqULB7jY+VVqcmEHYfzOeDzK383687cusZnYmvY3/vxrjJnzLUb4jIMmAoTnmJC1R1teuRBVN5vYUG3xTYOKLIrgP5fPD9Vsb8qj0dmzXky/FDrDHYmADxp9dQKpALfOC9TlU3uxlYUC18xvd6KzVd61SV977bwv0frCI3v4jTuzWnfUqCJQFjAsifR0OzcdoHBIgH2gNrgONdjCt4sjLw2UjcobLSSqaqtvx8mHveXc68Nbvol+oUiWufkhDssIyJOv48GurlvewpFHeDaxEF278u8b3+muidlM0NTpG4Rew5eIRJ5/Xg6pOtSJwxwVLlkcWq+o2IDKx8zzCVn1N2XXzTwMcRoTbvyaV1U6dI3CMX9SY1qQFtk6w+kDHB5E8bwW1eizFAP2CraxGFogkbgx1B2CssKualBRt4+r8/ctfwblw3uD2DO9m8wcaEAn/uCBK9XhfitBnMcCecIMuywWJuWLk1h/EzMlmxZT9nH9+Cc61InDEhpcJE4BlIlqiqfwpQPMH1zrXBjiDivPrVRh74cBVNGtTl71f2s0qhxoSgchOBiMR5KogODmRAQXXAxxOvhBaBjyMClBSJ63ZcIuf3ac2fR3SnSQPrEmpMKKrojiADpz3gOxGZBfwHOFSyUVVnuhxbaLjjx2BHEFYO5Rfy+Jw11IkV7jm3hxWJMyYM+NNGEA/swak+WjKeQIHISgSPdwl2BGFv/o+7uGvmcrbmHObak9OO3hUYY0JbRYmguafH0Ap+SQAlIq8M5KEdZdeJmxO4RY6c3AIemL2K6cuy6dDMKRLXP82KxBkTLir6pIsFGnJsAigReYnAl0Hjgh1BWNh9KJ+Pl2/jhtM6cvNQKxJnTLipKBFsU9XJAYskmJ7u5Xu91RYq184Decz6biu/PaXD0SJxTa0+kDFhqaJEED0Pd3Mit35ebVNVZnyzhQc+XMXhgiKGdm9B+5QESwLGhLGKEsHQgEURigbfGuwIQk7W3lzufnc5C37aTXq7pjxysRWJMyYSlJsIVHVvIAMJOfZY6BiFRcVc8dJi9h06wgPnH8+VA9sRY0XijIkI1i2mvEloDAAbdx+ibVID4mJjeOwSp0hcm6ZWJM6YSOLP5PWRzdckNLHxgY8jxBQUFfPC52s56+n5vLZoIwCDOqZYEjAmAtkdga+esKM/KLsuiqzYksOd0zNZtW0/5/ZqyYjerYIdkjHGRZYIfGk7INgRBM0/F27gwdmrSUqoyz+uOpFhPY8LdkjGGJdZIjDAL0Xijm/VmIv6tubec3vQuEGdYIdljAmA6E4EVl+Ig/mFPPbJD9SNjeHeET0Y0D6JAe2tPIQx0SS6G4t91ReqEz394uet2cnZT8/n9cWbUJy7AmNM9InuOwJfBvwu2BG4bt+hIzwwexUzv9lCp+YNmf5/gzixnc3LbEy0it5EUN60lFEwkGxf7hE+XbmDm4d04sYhnagXZ0XijIlmrj4aEpFhIrJGRNaKyAQf228TkVUikiki/xORdm7Gc4wom5Zy5/48psxfh6rSoVlDFo4fwm1ndbUkYIxxLxF45jt+ARgO9ACuEJEepXb7FkhX1d7AdOAxt+Ipw9e0lCldA3b6QFFV3lmSxdCnvuDJT39k455cAOsRZIw5ys1HQwOAtaq6HkBE3gLOB1aV7KCqn3vtvxi4ysV4KjeunMdFYSprby53zVzOl2t3M6B9Eo9c1MuKxBljynAzEbQGsryWs4GBFex/PfCxrw0iMhYYC5Camlpb8UW0kiJxP+cW8OAFPRk1INWKxBljfAqJxmIRuQpIB37ta7uqTgGmAKSnp1sfxwps2H2IVE+RuMcvOYF2yQ1o1aR+sMMyxoQwNxuLtwBtvZbbeNYdQ0TOAO4BRqpqvovx/CICK44WFBXz3P9+4uyn5/PqVxsBOLljsiUBY0yl3LwjWAJ0FpH2OAngcmCU9w4i0hd4ERimqjtdjOVYXz0XsFMFQmb2z9w5PZMfth/gvBNaMbKPFYkzxvjPtUSgqoUiMg6YA8QC01R1pYhMBpaq6izgcaAh8B8RAdisqiPdiumX4IrKrgvTHkPTvtzAg7NX0SyxHi9dk86ZPVoEOyRjTJhxtY1AVT8CPiq17j6v12e4ef4qCbMeQyVF4nq3acxl/dsyYXh3Gte3LqHGmKoLicZi478DeQU88vEP1IuL5b7zepCelkR6mhWJM8ZUX/QVnZsyJNgRVNvnP+zkrKfn82bGZuJixYrEGWNqRfTdEWz9puy6EJ+acu+hI0z+YCXvfbeVLi0a8rcrB9E31YrEGWNqR/QlgjCcmjLncAH/W72TW4Z25sbTO1E3Lvpu5Iwx7onCROBDCE5NuT0nj/e+28LvT+1A+5QEvpwwxBqDjTGusEQQYlSVt5Zk8fDs1RQUFzPs+ONIS0mwJGCMcY0lghCyac8hJsxYzqL1ezipQxKPXNSbNCsSZ4xxWXQlghmhO/tYYVExo176mpzDBTx8YS8u79/WisQZYwIiuhLBqvd8rAzuxCzrdh2knadI3JOXOkXiWja2+kDGmMCJru4nRQVl1w2+KfBxAEcKi/nrf39k2F/n89qiTQCc1CHZkoAxJuCi647AlyDMUfxd1s+Mn57Jmh0HOL9PKy7o2zrgMRhjTIkoSwTBH4k79csNPDR7Fc0T45l6bTpDu1uROGNMcEVZIgiekiJxfdo25vIBqUwY3o1G8dYl1BgTfJYIXLY/r4C/fPQD8XVimHje8ZzYLokT21mROGNM6IiuxuIA+++qHZz51Be8vWQzdeNirEicMSYkRc8dQVbg5hvYczCf+z9Yxazvt9LtuESmXJ3OCW2bBOz8xhhTFdGTCN68ImCnOpBXyOdrdvLHM7rwh9M6WpE4Y0xIi55EkLu77LoOtTc3wdafD/Put1u44bSOpKUksHDCEGsMNsaEhehJBL5c826N36K4WPl3xmYe+fgHioqVc3u1JC0lwZKAMSZsRE8iiK0HRfnHLtfQht2HmDAjk6837GVwp2T+cmFvUpMb1Ph9jTEmkKInEdRvCge3H7tcA4VFxVz18tfszyvgsYt785v0NohYkThjTPiJnkTgnQR8Lftp7c4DpCUnEBcbw9OX9aFdcgNaNArtqS6NMaYi1p3FT/mFRTw190eG/XUBr3qKxA1on2RJwBgT9qLnjqAGvtm8j/HTM/lp50Eu6tuai6xInDEmgkRPIpBY0KJjl/3w0vz1PPzxalo2iuef1/Xn9K7NXQrQGGOCI3oSgXcSANDiCncvLlZiYoR+7Zpw5cBUxg/rRqJ1CTXGRKDoSQR+yjlcwEOzV1G/Tiz3n9/TisQZYyJedDQWPz/Ar93mrNzOmU99wYxvtpBQL86KxBljokJ03BHs+ansujq/DPzafTCfie+vZPbybfRo2Yhpo/vTs3XjAAZojDHBEx2JoEEzOLTj2HUDfnf05cG8Qhb8tIs7zu7K2FM7UCc2Om6UjDEGouXRUINSo4jrJ7Gl/wSe/+wnVJW0lAS+umsoN57eyZKAMSbquPqpJyLDRGSNiKwVkQk+ttcTkbc9278WkTRXAjn889GXChwqiuGsp77ghc/XsWlPLgAN60XHzZExxpTmWiIQkVjgBWA40AO4QkR6lNrtemCfqnYCngYedSWY+GOf92cfrku/dk359I+nkpaS4MopjTEmXLh5RzAAWKuq61X1CPAWcH6pfc4HXvW8ng4MFTcqt7U8Ae/+P3Xb9uO1MQNom2SVQo0xxs1E0BrI8lrO9qzzuY+qFgI5QHLpNxKRsSKyVESW7tq1q+qR5O7GO7u0r59rlUKNMcYjLFpGVXWKqqaranqzZs2q/gbdf7kRkVLLxhgT7dxsId0CtPVabuNZ52ufbBGJAxoDe2o9kvTRzp+r33eSQMmyMcYYVxPBEqCziLTH+cC/HBhVap9ZwLXAIuAS4DN1azhv+mhLAMYY44NriUBVC0VkHDAHiAWmqepKEZkMLFXVWcBU4HURWQvsxUkWxhhjAsjVzvOq+hHwUal193m9zgN+42YMxhhjKhYWjcXGGGPcY4nAGGOinCUCY4yJcpYIjDEmykm4Tb4iIruATdU8PAXYXYvhhAO75uhg1xwdanLN7VTV54jcsEsENSEiS1U1PdhxBJJdc3Swa44Obl2zPRoyxpgoZ4nAGGOiXLQlginBDiAI7Jqjg11zdHDlmqOqjcAYY0xZ0XZHYIwxphRLBMYYE+UiMhGIyDARWSMia0Vkgo/t9UTkbc/2r0UkLQhh1io/rvk2EVklIpki8j8RaReMOGtTZdfstd/FIqIiEvZdDf25ZhG51PNvvVJE/h3oGGubH7/bqSLyuYh86/n9PicYcdYWEZkmIjtFZEU520VEnvX8fWSKSL8an1RVI+oHp+T1OqADUBf4HuhRap8bgH94Xl8OvB3suANwzacDDTyv/xAN1+zZLxGYDywG0oMddwD+nTsD3wJNPcvNgx13AK55CvAHz+sewMZgx13Daz4V6AesKGf7OcDHOBMungR8XdNzRuIdwQBgraquV9UjwFtA6bkpzwde9byeDgyV8J7EuNJrVtXPVTXXs7gYZ8a4cObPvzPAA8CjQF4gg3OJP9f8O+AFVd0HoKo7AxxjbfPnmhVo5HndGNgawPhqnarOx5mfpTznA6+pYzHQRERa1uSckZgIWgNZXsvZnnU+91HVQiAHSA5IdO7w55q9XY/zjSKcVXrNnlvmtqo6O5CBuciff+cuQBcRWSgii0VkWMCic4c/1zwJuEpEsnHmP7kpMKEFTVX/v1fK1YlpTOgRkauAdODXwY7FTSISAzwFjA5yKIEWh/N46DScu775ItJLVX8OZlAuuwJ4RVWfFJGTcWY97KmqxcEOLFxE4h3BFqCt13Ibzzqf+4hIHM7t5J6AROcOf64ZETkDuAcYqar5AYrNLZVdcyLQE5gnIhtxnqXOCvMGY3/+nbOBWapaoKobgB9xEkO48uearwfeAVDVRUA8TnG2SOXX//eqiMREsAToLCLtRaQuTmPwrFL7zAKu9by+BPhMPa0wYarSaxaRvsCLOEkg3J8bQyXXrKo5qpqiqmmqmobTLjJSVZcGJ9xa4c/v9ns4dwOISArOo6L1AYyxtvlzzZuBoQAi0h0nEewKaJSBNQu4xtN76CQgR1W31eQNI+7RkKoWisg4YA5Oj4NpqrpSRCYDS1V1FjAV5/ZxLU6jzOXBi7jm/Lzmx4GGwH887eKbVXVk0IKuIT+vOaL4ec1zgLNEZBVQBNyhqmF7t+vnNd8OvCQif8RpOB4dzl/sRORNnGSe4mn3mAjUAVDVf+C0g5wDrAVygetqfM4w/vsyxhhTCyLx0ZAxxpgqsERgjDFRzhKBMcZEOUsExhgT5SwRGGNMlLNEYEKSiBSJyHdeP2kV7HuwFs73iohs8JzrG88I1aq+x8si0sPz+u5S276qaYye9yn5e1khIh+ISJNK9u8T7tU4jfus+6gJSSJyUFUb1va+FbzHK8CHqjpdRM4CnlDV3jV4vxrHVNn7isirwI+q+lAF+4/Gqbo6rrZjMZHD7ghMWBCRhp55FL4RkeUiUqbSqIi0FJH5Xt+YT/GsP0tEFnmO/Y+IVPYBPR/o5Dn2Ns97rRCRWz3rEkRktoh871l/mWf9PBFJF5FHgPqeON7wbDvo+fMtETnXK+ZXROQSEYkVkcdFZImnxvzv/fhrWYSn2JiIDPBc47ci8pWIdPWMxJ0MXOaJ5TJP7NNEJMOzr6+KrSbaBLv2tv3Yj68fnFGx33l+3sUZBd/Isy0FZ1RlyR3tQc+ftwP3eF7H4tQbSsH5YE/wrB8P3OfjfK8Al3he/wb4GjgRWA4k4IzKXgn0BS4GXvI6trHnz3l45jwoiclrn5IYLwRe9byui1NFsj4wFrjXs74esBRo7yPOg17X9x9gmGe5ERDneX0GMMPzejTwvNfxDwNXeV43walFlBDsf2/7Ce5PxJWYMBHjsKr2KVkQkTrAwyJyKlCM8024BbDd65glwDTPvu+p6nci8mucyUoWekpr1MX5Ju3L4yJyL06dmutx6te8q6qHPDHMBE4BPgGeFJFHcR4nLajCdX0MPCMi9YBhwHxVPex5HNVbRC7x7NcYp1jchlLH1xeR7zzXvxqY67X/qyLSGafMQp1yzn8WMFJE/uRZjgdSPe9lopQlAhMurgSaASeqaoE4FUXjvXdQ1fmeRHEu8IqIPAXsA+aq6hV+nOMOVZ1esiAiQ33tpKo/ijPXwTnAgyLyP1Wd7M9FqGqeiMwDzgYuw5loBZzZpm5S1TmVvMVhVe0jIg1w6u/cCDyLMwHP56p6oadhfV45xwtwsaqu8SdeEx2sjcCEi8bATk8SOB0oM+eyOPMw71DVl4CXcab7WwwMFpGSZ/4JItLFz3MuAC4QkQYikoDzWGeBiLQCclX1XzjF/HzNGVvguTPx5W2cQmEldxfgfKj/oeQYEeniOadP6sw2dzNwu/xSSr2kFPFor10P4DwiKzEHuEk8t0fiVKU1Uc4SgQkXbwDpIrIcuAb4wcc+pwHfi8i3ON+2n1HVXTgfjG+KSCbOY6Fu/pxQVb/BaTvIwGkzeFlVvwV6ARmeRzQTgQd9HD4FyCxpLC7lU5yJgf6rzvSL4CSuVcA34kxa/iKV3LF7YsnEmZjlMeAvnmv3Pu5zoEdJYzHOnUMdT2wrPcsmyln3UWOMiXJ2R2CMMVHOEoExxkQ5SwTGGBPlLBEYY0yUs0RgjDFRzhKBMcZEOUsExhgT5f4fLUenK6lCtDsAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot\n",
    "# plot the roc curve for the model\n",
    "pyplot.plot([0,1], [0,1], linestyle='--', label='No Skill')\n",
    "pyplot.plot(fpr, tpr, marker='.', label='NN')\n",
    "# axis labels\n",
    "pyplot.xlabel('False Positive Rate')\n",
    "pyplot.ylabel('True Positive Rate')\n",
    "pyplot.legend()\n",
    "# show the plot\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Kappa metric for test\n",
    "from sklearn.metrics import cohen_kappa_score\n",
    "\n",
    "cohen_kappa_score(y_true,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Train data reeults\n",
    "train_predicts = model.predict(X_train)\n",
    "train_predicts\n",
    "\n",
    "train_evaluate = X_train.copy()\n",
    "train_evaluate['subscribed'] = y_train\n",
    "train_evaluate['prediction_score'] = train_predicts\n",
    "train_evaluate['prediction_label'] = np.where(train_evaluate.prediction_score > 0.5, 1, 0)\n",
    "train_evaluate\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix,ConfusionMatrixDisplay\n",
    "import matplotlib.pyplot as plt\n",
    "y_train_true = train_evaluate['subscribed']\n",
    "y_train_pred = train_evaluate['prediction_label']\n",
    "cm = confusion_matrix(y_train_true, y_train_pred)\n",
    "\n",
    "# disp.plot()\n",
    "\n",
    "# plt.show()\n",
    "\n",
    "# cm\n",
    "\n",
    "# Plot the confusion matrix\n",
    "plt.imshow(cm, interpolation='nearest', cmap=plt.cm.Blues)\n",
    "plt.title('Confusion Matrix')\n",
    "plt.colorbar()\n",
    "tick_marks = np.arange(len(np.unique(y_true)))\n",
    "plt.xticks(tick_marks, np.unique(y_true))\n",
    "plt.yticks(tick_marks, np.unique(y_true))\n",
    "plt.xlabel('Predicted Label')\n",
    "plt.ylabel('True Label')\n",
    "\n",
    "# Fill the confusion matrix cells with values\n",
    "thresh = cm.max() / 2.0\n",
    "for i, j in np.ndindex(cm.shape):\n",
    "    plt.text(j, i, format(cm[i, j], 'd'), horizontalalignment=\"center\",\n",
    "             color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "# Show the plot\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "print(classification_report(y_train, y_train_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import RocCurveDisplay\n",
    "import matplotlib.pyplot as plt\n",
    "roc_auc_score(y_train, train_predicts)\n",
    "\n",
    "#RocCurveDisplay.from_predictions(y_true, y_pred)\n",
    "\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cohen_kappa_score(y_train_true,y_train_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#youden j = 0.39\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}