DCAuth.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 🔐 Authentication"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 📚 Loading libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import shap\n",
    "import sys\n",
    "import time\n",
    "import tsfresh\n",
    "\n",
    "from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis\n",
    "from sklearn.ensemble import AdaBoostClassifier, RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, f1_score, precision_score, recall_score\n",
    "from sklearn.model_selection import GridSearchCV, train_test_split\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "from utils.const import *\n",
    "from utils.helperFunctions import *\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "if not sys.warnoptions:\n",
    "    warnings.simplefilter(\"ignore\")\n",
    "    os.environ[\"PYTHONWARNINGS\"] = \"ignore\"  # Also affect subprocesses"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 📍 Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Listing IDs\n",
    "ids = ['id1', 'id2', 'id3']\n",
    "# Choose what ID to process\n",
    "id = 'id2'\n",
    "ids_remove = [x for x in ids if x != id]\n",
    "\n",
    "# Filter features and keep only relevant ones\n",
    "filterFeatures = True\n",
    "\n",
    "# Undersampling\n",
    "fairUndersampling = False       # Each class same number\n",
    "targetedUndersampling = True    # Downsample most frequent class\n",
    "customBalance = False           # Downsample by specifying number of samples for each label\n",
    "\n",
    "# Choose whether to separate Charge and Discharge cycling or not\n",
    "separate = False\n",
    "\n",
    "# If True, perform authentication. If False, perform identification\n",
    "# - Authentication: binary classification, unbalanced\n",
    "# - Identificatiom: multiclass classification, balanced\n",
    "authentication = True\n",
    "\n",
    "# Results names and folders\n",
    "if not os.path.exists(RESULTS):\n",
    "    os.mkdir(RESULTS)\n",
    "    os.mkdir(FIGURES)\n",
    "\n",
    "if authentication:\n",
    "    saveBase = id.upper() + '_AUTH'\n",
    "else:\n",
    "    saveBase = id.upper() + '_IDENT'\n",
    "\n",
    "imageFolder = os.path.join(FIGURES, saveBase)\n",
    "\n",
    "if separate:\n",
    "    saveBase += '_SEPARATE'\n",
    "    imageFolder = os.path.join(FIGURES, saveBase)\n",
    "\n",
    "if not os.path.exists(imageFolder):\n",
    "    os.makedirs(imageFolder)\n",
    "\n",
    "if not os.path.exists(COMPLEXITY):\n",
    "    os.mkdir(COMPLEXITY)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 🤖 Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "names = [\n",
    "    'AdaBoost',\n",
    "    'Decision Tree',\n",
    "    'Gaussian Naive Bayes',\n",
    "    'Nearest Neighbors',\n",
    "    'Neural Network',\n",
    "    'Quadratic Discriminant Analysis',\n",
    "    'Random Forest',\n",
    "    'Support Vector Machine'\n",
    "]\n",
    "\n",
    "classifiers = [\n",
    "    AdaBoostClassifier(random_state=SEED),\n",
    "    DecisionTreeClassifier(random_state=SEED),\n",
    "    GaussianNB(),\n",
    "    KNeighborsClassifier(),\n",
    "    MLPClassifier(random_state=SEED),\n",
    "    QuadraticDiscriminantAnalysis(),\n",
    "    RandomForestClassifier(random_state=SEED),\n",
    "    SVC(random_state=SEED),\n",
    "]\n",
    "\n",
    "parameters = [\n",
    "    # AdaBoostClassifier\n",
    "    {\n",
    "        'n_estimators': [50, 100, 150, 200]\n",
    "    },\n",
    "    # DecisionTreeClassifier\n",
    "    {\n",
    "        'criterion': ['gini', 'entropy', 'log_loss'],\n",
    "        'max_depth': np.arange(3, 20)\n",
    "    },\n",
    "    # GaussianNB\n",
    "    {\n",
    "        'var_smoothing': np.logspace(0, -9, num=100)\n",
    "    },\n",
    "    # KNeighborsClassifier\n",
    "    {\n",
    "        'n_neighbors': list(range(1, 20)),\n",
    "        'weights': ['uniform', 'distance']\n",
    "    },\n",
    "    # MLPClassifier\n",
    "    {\n",
    "        'hidden_layer_sizes': [(50, ), (100, ), (200, )],\n",
    "        'activation': ['tanh', 'relu'],\n",
    "        'solver': ['adam', 'sgd']\n",
    "    },\n",
    "    # QuadraticDiscriminantAnalysis\n",
    "    {\n",
    "        'reg_param': [0.1, 0.2, 0.3, 0.4, 0.5]\n",
    "    },\n",
    "    # RandomForestClassifier\n",
    "    {\n",
    "        'criterion': ['gini', 'entropy', 'log_loss'],\n",
    "        'n_estimators': [100, 200, 300, 400, 500]\n",
    "    },\n",
    "    # SVC\n",
    "    {\n",
    "        'kernel': ['linear', 'rbf'],\n",
    "        'C': np.arange(1, 3, 1),\n",
    "        'gamma': np.arange(0.25, 1, 0.25)\n",
    "    },\n",
    "]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 📚 Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not separate:\n",
    "    dff = []\n",
    "    for dataset in DATASETS:\n",
    "        dir = os.path.join(PROCESSED, dataset)\n",
    "        for file in os.listdir(dir):\n",
    "            if file.split('.')[-1] == 'parquet':\n",
    "                df = pd.read_parquet(os.path.join(dir, file))\n",
    "                dff.append(df)\n",
    "\n",
    "        df = pd.concat(dff)\n",
    "else:\n",
    "    dff_charge = []\n",
    "    dff_discharge = []\n",
    "    for dataset in DATASETS:\n",
    "        dir = os.path.join(PROCESSED, dataset)\n",
    "        for file in os.listdir(dir):\n",
    "            if file.split('.')[-1] == 'parquet':\n",
    "                df = pd.read_parquet(os.path.join(dir, file))\n",
    "                if file.split('.')[0] == 'charge':\n",
    "                    dff_charge.append(df)\n",
    "                elif file.split('.')[0] == 'discharge':\n",
    "                    dff_discharge.append(df)\n",
    "\n",
    "    df_charge = pd.concat(dff_charge)\n",
    "    df_discharge = pd.concat(dff_discharge)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not separate:\n",
    "    fig, axs = plt.subplots(1, 2, figsize=(8, 4))\n",
    "    axs = axs.ravel()\n",
    "\n",
    "    df['id1'].value_counts().sort_index().plot(\n",
    "        kind='bar', title='ID1 Distribution', xlabel='IDs', ylabel='Occurences', ax=axs[0])\n",
    "    df['id2'].value_counts().sort_index().plot(\n",
    "        kind='bar', title='ID2 Distribution', xlabel='IDs', ylabel='Occurences', ax=axs[1])\n",
    "\n",
    "else:\n",
    "    fig, axs = plt.subplots(2, 2, figsize=(8, 8))\n",
    "    axs = axs.ravel()\n",
    "\n",
    "    df_charge['id1'].value_counts().sort_index().plot(\n",
    "        kind='bar', title='ID1 Distribution on Charge', xlabel='IDs', ylabel='Occurences', ax=axs[0])\n",
    "    df_charge['id2'].value_counts().sort_index().plot(\n",
    "        kind='bar', title='ID2 Distribution on Charge', xlabel='IDs', ylabel='Occurences', ax=axs[1])\n",
    "    df_discharge['id1'].value_counts().sort_index().plot(\n",
    "        kind='bar', title='ID1 Distribution on Discharge', xlabel='IDs', ylabel='Occurences', ax=axs[2])\n",
    "    df_discharge['id2'].value_counts().sort_index().plot(\n",
    "        kind='bar', title='ID2 Distribution on Discharge', xlabel='IDs', ylabel='Occurences', ax=axs[3])\n",
    "\n",
    "plt.tight_layout()\n",
    "saveDistPath = os.path.join(imageFolder, saveBase)\n",
    "saveDistPath += '_unbalancedDistribution.pdf'\n",
    "plt.savefig(saveDistPath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if targetedUndersampling:\n",
    "    if not separate:\n",
    "        df_x = df.drop(id, axis=1)\n",
    "        df_x = df_x.drop(ids_remove, axis=1)\n",
    "        \n",
    "        X_resampled, y_resampled = RandomUnderSampler(random_state=SEED).fit_resample(df_x, df[id])\n",
    "\n",
    "        X_resampled[id] = y_resampled\n",
    "        df = X_resampled\n",
    "\n",
    "        df[id].value_counts().sort_index().plot(\n",
    "            kind='bar', title='ID Distribution', xlabel='IDs', ylabel='Occurences')\n",
    "    else:\n",
    "        dfc_x = df_charge.drop(id, axis=1)\n",
    "        dfd_x = df_discharge.drop(id, axis=1)\n",
    "\n",
    "        dfc_x = dfc_x.drop(ids_remove, axis=1)\n",
    "        dfd_x = dfd_x.drop(ids_remove, axis=1)\n",
    "        \n",
    "        Xc_resampled, yc_resampled = RandomUnderSampler(random_state=SEED).fit_resample(dfc_x, df_charge[id])\n",
    "        Xd_resampled, yd_resampled = RandomUnderSampler(random_state=SEED).fit_resample(dfd_x, df_discharge[id])\n",
    "\n",
    "        Xc_resampled[id] = yc_resampled\n",
    "        Xd_resampled[id] = yd_resampled\n",
    "        \n",
    "        df_charge = Xc_resampled\n",
    "        df_discharge = Xd_resampled\n",
    "\n",
    "        fig, axs = plt.subplots(1, 2, figsize=(8, 4))\n",
    "        axs = axs.ravel()\n",
    "\n",
    "        df_charge[id].value_counts().sort_index().plot(\n",
    "            kind='bar', title='ID Distribution on Charge', xlabel='IDs', ylabel='Occurences', ax=axs[0])\n",
    "        df_discharge[id].value_counts().sort_index().plot(\n",
    "            kind='bar', title='ID Distribution on Discharge', xlabel='IDs', ylabel='Occurences', ax=axs[1])\n",
    "\n",
    "saveBalPath = os.path.join(imageFolder, saveBase)\n",
    "saveBalPath += '_balancedDistribution.pdf'\n",
    "plt.savefig(saveBalPath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not separate:\n",
    "    beforeFeat = df.shape[1]\n",
    "    tsfresh.utilities.dataframe_functions.impute(df)\n",
    "\n",
    "    if filterFeatures:\n",
    "        df = tsfresh.select_features(df, df[id])\n",
    "        afterFeat = df.shape[1]\n",
    "\n",
    "        print(f'[🔥 FILTER]\\n\\tBefore: {beforeFeat}\\n\\tAfter: {afterFeat}')\n",
    "else:\n",
    "    beforeFeat_c = df_charge.shape[1]\n",
    "    beforeFeat_d = df_discharge.shape[1]\n",
    "\n",
    "    tsfresh.utilities.dataframe_functions.impute(df_charge)\n",
    "    tsfresh.utilities.dataframe_functions.impute(df_discharge)\n",
    "\n",
    "    if filterFeatures:\n",
    "        df_charge = tsfresh.select_features(df_charge, df_charge[id])\n",
    "        df_discharge = tsfresh.select_features(df_discharge, df_discharge[id])\n",
    "        \n",
    "        afterFeat_c = df_charge.shape[1]\n",
    "        afterFeat_d = df_discharge.shape[1]\n",
    "\n",
    "        print(f'[🔥 CHARGE]\\n\\tBefore: {beforeFeat_c}\\n\\tAfter: {afterFeat_c}')\n",
    "        print()\n",
    "        print(f'[🔥 DISCHARGE]\\n\\tBefore: {beforeFeat_d}\\n\\tAfter: {afterFeat_d}')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 💪 Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not separate:\n",
    "    # Loading labels\n",
    "    labels = df[id][:, np.newaxis]\n",
    "\n",
    "    if authentication:\n",
    "\n",
    "        # Lists of datasets\n",
    "        X_trains = []\n",
    "        X_tests = []\n",
    "        Y_trains = []\n",
    "        Y_tests = []\n",
    "        all_balances = []\n",
    "        \n",
    "        for balance in BALANCES:\n",
    "            # Translating to authentication, i.e., taking only one label\n",
    "            # Saving different dataset, one for each label\n",
    "            for label in np.unique(labels):\n",
    "                labels_auth = []\n",
    "                for l in labels:\n",
    "                    if l == label:\n",
    "                        labels_auth.append(1)\n",
    "                    else:\n",
    "                        labels_auth.append(0)\n",
    "\n",
    "                labels_auth = np.array(labels_auth)\n",
    "\n",
    "                l0 = 0\n",
    "                l1 = 0\n",
    "                for l in labels_auth:\n",
    "                    if l == 0:\n",
    "                        l0 += 1\n",
    "                    else:\n",
    "                        l1 += 1\n",
    "\n",
    "                # How many 0s would I need by looking at the 1 class\n",
    "                requested_0 = int(l1 / balance[1] * balance[0])\n",
    "                # How many 1s would I need by looking at the 0 class\n",
    "                requested_1 = int(l0 / balance[0] * balance[1])\n",
    "\n",
    "                if requested_0 < l0:\n",
    "                    num_samples = {\n",
    "                        0: requested_0,\n",
    "                        1: l1\n",
    "                    }\n",
    "                if requested_1 < l1:\n",
    "                    num_samples = {\n",
    "                        0: l0,\n",
    "                        1: requested_1\n",
    "                    }\n",
    "\n",
    "                # Loading features\n",
    "                features = df.drop(id, axis=1)\n",
    "\n",
    "                # Dataset balancing\n",
    "                features, labels_auth = RandomUnderSampler(sampling_strategy=num_samples,\n",
    "                    random_state=SEED).fit_resample(features, labels_auth)\n",
    "\n",
    "                # Train and test split\n",
    "                X_train, X_test, Y_train, Y_test = train_test_split(\n",
    "                    features, labels_auth, test_size=0.2, random_state=SEED)\n",
    "\n",
    "                cols = []\n",
    "                for col in X_train.columns:\n",
    "                    cols.append(col.replace('dQ/dV__', ''))\n",
    "\n",
    "                X_train.columns = cols\n",
    "                X_test.columns = cols\n",
    "\n",
    "                X_trains.append(X_train)\n",
    "                X_tests.append(X_test)\n",
    "                Y_trains.append(Y_train)\n",
    "                Y_tests.append(Y_test)\n",
    "                all_balances.append(balance)\n",
    "    else:\n",
    "        # Loading features\n",
    "        features = df.drop(id, axis=1)\n",
    "\n",
    "        # Train and test split\n",
    "        X_train, X_test, Y_train, Y_test = train_test_split(\n",
    "            features, labels, test_size=0.2, random_state=SEED)\n",
    "\n",
    "        cols = []\n",
    "        for col in X_train.columns:\n",
    "            cols.append(col.replace('dQ/dV__', ''))\n",
    "\n",
    "        X_train.columns = cols\n",
    "        X_test.columns = cols\n",
    "else:\n",
    "    # Loading labels\n",
    "    labels_c = df_charge[id][:, np.newaxis]\n",
    "    labels_d = df_discharge[id][:, np.newaxis]\n",
    "\n",
    "    if authentication:\n",
    "\n",
    "        # Lists of datasets\n",
    "        X_trains_c = []\n",
    "        X_tests_c = []\n",
    "        Y_trains_c = []\n",
    "        Y_tests_c = []\n",
    "\n",
    "        X_trains_d = []\n",
    "        X_tests_d = []\n",
    "        Y_trains_d = []\n",
    "        Y_tests_d = []\n",
    "\n",
    "        all_balances_c = []\n",
    "        all_balances_d = []\n",
    "\n",
    "        for balance in BALANCES:\n",
    "            # Translating to authentication, i.e., taking only one label\n",
    "            # Saving different dataset, one for each label\n",
    "            for label in np.unique(labels_c):\n",
    "                labels_auth_c = []\n",
    "                for l in labels_c:\n",
    "                    if l == label:\n",
    "                        labels_auth_c.append(1)\n",
    "                    else:\n",
    "                        labels_auth_c.append(0)\n",
    "\n",
    "                labels_auth_c = np.array(labels_auth_c)\n",
    "\n",
    "                l0_c = 0\n",
    "                l1_c = 0\n",
    "                for l in labels_auth_c:\n",
    "                    if l == 0:\n",
    "                        l0_c += 1\n",
    "                    else:\n",
    "                        l1_c += 1\n",
    "\n",
    "                # How many 0s would I need by looking at the 1 class\n",
    "                requested_0_c = int(l1_c / balance[1] * balance[0])\n",
    "                # How many 1s would I need by looking at the 0 class\n",
    "                requested_1_c = int(l0_c / balance[0] * balance[1])\n",
    "\n",
    "                if requested_0_c < l0_c:\n",
    "                    num_samples_c = {\n",
    "                        0: requested_0_c,\n",
    "                        1: l1_c\n",
    "                    }\n",
    "                if requested_1_c < l1_c:\n",
    "                    num_samples_c = {\n",
    "                        0: l0_c,\n",
    "                        1: requested_1_c\n",
    "                    }\n",
    "\n",
    "                # Loading features\n",
    "                features_c = df_charge.drop(id, axis=1)\n",
    "\n",
    "                # Dataset balancing\n",
    "                features_c, labels_auth_c = RandomUnderSampler(sampling_strategy=num_samples_c,\n",
    "                    random_state=SEED).fit_resample(features_c, labels_auth_c)\n",
    "\n",
    "                # Train and test split\n",
    "                X_train_c, X_test_c, Y_train_c, Y_test_c = train_test_split(\n",
    "                    features_c, labels_auth_c, test_size=0.2, random_state=SEED)\n",
    "\n",
    "                cols = []\n",
    "                for col in X_train_c.columns:\n",
    "                    cols.append(col.replace('dQ/dV__', ''))\n",
    "\n",
    "                X_train_c.columns = cols\n",
    "                X_test_c.columns = cols\n",
    "\n",
    "                X_trains_c.append(X_train_c)\n",
    "                X_tests_c.append(X_test_c)\n",
    "                Y_trains_c.append(Y_train_c)\n",
    "                Y_tests_c.append(Y_test_c)\n",
    "                all_balances_c.append(balance)\n",
    "\n",
    "            for label in np.unique(labels_d):\n",
    "                labels_auth_d = []\n",
    "                for l in labels_d:\n",
    "                    if l == label:\n",
    "                        labels_auth_d.append(1)\n",
    "                    else:\n",
    "                        labels_auth_d.append(0)\n",
    "\n",
    "                labels_auth_d = np.array(labels_auth_d)\n",
    "\n",
    "                l0_d = 0\n",
    "                l1_d = 0\n",
    "                for l in labels_auth_d:\n",
    "                    if l == 0:\n",
    "                        l0_d += 1\n",
    "                    else:\n",
    "                        l1_d += 1\n",
    "\n",
    "                # How many 0s would I need by looking at the 1 class\n",
    "                requested_0_d = int(l1_d / balance[1] * balance[0])\n",
    "                # How many 1s would I need by looking at the 0 class\n",
    "                requested_1_d = int(l0_d / balance[0] * balance[1])\n",
    "\n",
    "                if requested_0_d < l0_d:\n",
    "                    num_samples_d = {\n",
    "                        0: requested_0_d,\n",
    "                        1: l1_d\n",
    "                    }\n",
    "                if requested_1_d < l1_d:\n",
    "                    num_samples_d = {\n",
    "                        0: l0_d,\n",
    "                        1: requested_1_d\n",
    "                    }\n",
    "\n",
    "                # Loading features\n",
    "                features_d = df_discharge.drop(id, axis=1)\n",
    "\n",
    "                # Dataset balancing\n",
    "                features_d, labels_auth_d = RandomUnderSampler(sampling_strategy=num_samples_d,\n",
    "                    random_state=SEED).fit_resample(features_d, labels_auth_d)\n",
    "\n",
    "                # Train and test split\n",
    "                X_train_d, X_test_d, Y_train_d, Y_test_d = train_test_split(\n",
    "                    features_d, labels_auth_d, test_size=0.2, random_state=SEED)\n",
    "\n",
    "                cols = []\n",
    "                for col in X_train_d.columns:\n",
    "                    cols.append(col.replace('dQ/dV__', ''))\n",
    "\n",
    "                X_train_d.columns = cols\n",
    "                X_test_d.columns = cols\n",
    "\n",
    "                X_trains_d.append(X_train_d)\n",
    "                X_tests_d.append(X_test_d)\n",
    "                Y_trains_d.append(Y_train_d)\n",
    "                Y_tests_d.append(Y_test_d)\n",
    "                all_balances_d.append(balance)\n",
    "\n",
    "    else:\n",
    "        # Loading features\n",
    "        features_c = df_charge.drop(id, axis=1)\n",
    "        features_d = df_discharge.drop(id, axis=1)\n",
    "\n",
    "        # Train and test split\n",
    "        X_train_c, X_test_c, Y_train_c, Y_test_c = train_test_split(\n",
    "            features_c, labels_c, test_size=0.2, random_state=SEED)\n",
    "        X_train_d, X_test_d, Y_train_d, Y_test_d = train_test_split(\n",
    "            features_d, labels_d, test_size=0.2, random_state=SEED)\n",
    "        \n",
    "        cols_c = []\n",
    "        cols_d = []\n",
    "        for col in X_train_c.columns:\n",
    "            cols_c.append(col.replace('dQ/dV__', ''))\n",
    "        for col in X_train_d.columns:\n",
    "            cols_d.append(col.replace('dQ/dV__', ''))\n",
    "\n",
    "        X_train_c.columns = cols_c\n",
    "        X_test_c.columns = cols_c\n",
    "        X_train_d.columns = cols_d\n",
    "        X_test_d.columns = cols_d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not separate:\n",
    "    train_scores = []\n",
    "\n",
    "    accuracy_scores = []\n",
    "    precision_scores = []\n",
    "    recall_scores = []\n",
    "    f1_scores = []\n",
    "\n",
    "    best_params = []\n",
    "\n",
    "    model_balances = []\n",
    "\n",
    "    fars = []\n",
    "    frrs = []\n",
    "\n",
    "    complexity = []\n",
    "else:\n",
    "    train_scores_c = []\n",
    "    train_scores_d = []\n",
    "\n",
    "    accuracy_scores_c = []\n",
    "    precision_scores_c = []\n",
    "    recall_scores_c = []\n",
    "    f1_scores_c = []\n",
    "    accuracy_scores_d = []\n",
    "    precision_scores_d = []\n",
    "    recall_scores_d = []\n",
    "    f1_scores_d = []\n",
    "    \n",
    "    best_params_c = []\n",
    "    best_params_d = []\n",
    "\n",
    "    model_balances_c = []\n",
    "    model_balances_d = []\n",
    "\n",
    "    fars_c = []\n",
    "    fars_d = []\n",
    "    frrs_c = []\n",
    "    frrs_d = []\n",
    "\n",
    "    complexity_c = []\n",
    "    complexity_d = []\n",
    "\n",
    "# Iterate over classifiers\n",
    "for name, clf, param in zip(names, classifiers, parameters):\n",
    "    if not separate:\n",
    "        if authentication:\n",
    "            score_trains = []\n",
    "            accuracy_tests = []\n",
    "            precision_tests = []\n",
    "            recall_tests = []\n",
    "            f1_tests = []\n",
    "            fars_tests = []\n",
    "            frrs_tests = []\n",
    "            for i, (X_train, X_test, Y_train, Y_test, balance) in enumerate(zip(X_trains, X_tests, Y_trains, Y_tests, all_balances)):\n",
    "                # Defining GridSearch\n",
    "                grid = GridSearchCV(clf, param, n_jobs=-1, verbose=0)\n",
    "                print(f'[🤖 MODEL] {name} ({i+1}/{len(X_trains)})', end='\\r')\n",
    "                # Fitting the model\n",
    "                grid.fit(X_train, Y_train)\n",
    "                # Training score\n",
    "                score_trains.append(grid.best_estimator_.score(X_train, Y_train))\n",
    "                # Test scores\n",
    "                start_pred = time.time()\n",
    "                Y_pred = grid.best_estimator_.predict(X_test)\n",
    "                end_pred = time.time()\n",
    "\n",
    "                pred_time = end_pred - start_pred\n",
    "                size = sys.getsizeof(pickle.dumps(grid.best_estimator_))\n",
    "\n",
    "                complexity.append({\n",
    "                    'Model': name,\n",
    "                    'Prediction_Time': pred_time,\n",
    "                    'Size': size\n",
    "                })\n",
    "\n",
    "                accuracy_tests.append(accuracy_score(Y_test, Y_pred))\n",
    "                precision_tests.append(precision_score(Y_test, Y_pred))\n",
    "                recall_tests.append(recall_score(Y_test, Y_pred))\n",
    "                f1_tests.append(f1_score(Y_test, Y_pred))\n",
    "                # FARs and FRRs\n",
    "                cm = confusion_matrix(y_true=Y_test, y_pred=Y_pred)\n",
    "                tp = cm[0][0]\n",
    "                tn = cm[1][1]\n",
    "                fp = cm[0][1]\n",
    "                fn = cm[1][0]\n",
    "                far = fp/(fp+tn)\n",
    "                if math.isnan(far):\n",
    "                    far = 0\n",
    "                frr = fn/(fn+tp)\n",
    "                if math.isnan(frr):\n",
    "                    frr = 0\n",
    "                fars_tests.append(far)\n",
    "                frrs_tests.append(frr)\n",
    "\n",
    "                # Save results by doing the mean on each balance value\n",
    "                if (i+1) % len(np.unique(labels)) == 0:\n",
    "                    print()\n",
    "                    print(f'\\t[⚖️ BALANCE] {balance[0]}/{balance[1]}')\n",
    "                    print(f'\\t\\t[💪 TRAIN]\\t{round(np.mean(score_trains), 3)}')\n",
    "                    print(f'\\t\\t[📊 ACCURACY]\\t{round(np.mean(accuracy_tests), 3)}')\n",
    "                    print(f'\\t\\t[📊 PRECISION]\\t{round(np.mean(precision_tests), 3)}')\n",
    "                    print(f'\\t\\t[📊 RECALL]\\t{round(np.mean(recall_tests), 3)}')\n",
    "                    print(f'\\t\\t[📊 F1 SCORE]\\t{round(np.mean(f1_tests), 3)}')\n",
    "                    print(f'\\t\\t[🔐 FAR]\\t{round(np.mean(fars_tests), 3)}')\n",
    "                    print(f'\\t\\t[🔐 FRR]\\t{round(np.mean(frrs_tests), 3)}\\n')\n",
    "                    # Saving results\n",
    "                    train_scores.append(round(np.mean(score_trains), 3))\n",
    "                    accuracy_scores.append(round(np.mean(accuracy_tests), 3))\n",
    "                    precision_scores.append(round(np.mean(precision_tests), 3))\n",
    "                    recall_scores.append(round(np.mean(recall_tests), 3))\n",
    "                    f1_scores.append(round(np.mean(f1_tests), 3))\n",
    "                    fars.append(round(np.mean(fars_tests), 3))\n",
    "                    frrs.append(round(np.mean(frrs_tests), 3))\n",
    "                    best_params.append(grid.best_params_)\n",
    "                    model_balances.append(balance)\n",
    "                    # Resetting results\n",
    "                    score_trains = []\n",
    "                    accuracy_tests = []\n",
    "                    precision_tests = []\n",
    "                    recall_tests = []\n",
    "                    f1_tests = []\n",
    "                    fars_tests = []\n",
    "                    frrs_tests = []\n",
    "            print('-' * 50)\n",
    "            print()\n",
    "\n",
    "        else:\n",
    "            print(f'[🤖 MODEL] {name}')\n",
    "            # Defining GridSearch\n",
    "            grid = GridSearchCV(clf, param, n_jobs=-1, verbose=0)\n",
    "            # Fitting the model\n",
    "            grid.fit(X_train, Y_train)\n",
    "            # Training score\n",
    "            score_train = grid.best_estimator_.score(X_train, Y_train)\n",
    "            print(f'\\t[💪 TRAIN]\\t{round(score_train, 3)}')\n",
    "            # Test scores\n",
    "            start_pred = time.time()\n",
    "            Y_pred = grid.best_estimator_.predict(X_test)\n",
    "            end_pred = time.time()\n",
    "\n",
    "            pred_time = end_pred - start_pred\n",
    "            size = sys.getsizeof(pickle.dumps(grid.best_estimator_))\n",
    "\n",
    "            complexity.append({\n",
    "                'Model': name,\n",
    "                'Prediction_Time': pred_time,\n",
    "                'Size': size\n",
    "            })\n",
    "            accuracy = accuracy_score(Y_test, Y_pred)\n",
    "            precision = precision_score(Y_test, Y_pred, average='macro')\n",
    "            recall = recall_score(Y_test, Y_pred, average='macro')\n",
    "            f1 = f1_score(Y_test, Y_pred, average='macro')\n",
    "\n",
    "            print(f'\\t[📊 ACCURACY]\\t{round(accuracy, 3)}')\n",
    "            print(f'\\t[📊 PRECISION]\\t{round(precision, 3)}')\n",
    "            print(f'\\t[📊 RECALL]\\t{round(recall, 3)}')\n",
    "            print(f'\\t[📊 F1 SCORE]\\t{round(f1, 3)}\\n')\n",
    "\n",
    "            train_scores.append(round(score_train, 3))\n",
    "            accuracy_scores.append(round(accuracy, 3))\n",
    "            precision_scores.append(round(precision, 3))\n",
    "            recall_scores.append(round(recall, 3))\n",
    "            f1_scores.append(round(f1, 3))\n",
    "            best_params.append(grid.best_params_)\n",
    "\n",
    "        # Feature importance for Random Forest\n",
    "        if name == 'Random Forest':\n",
    "            # Confusion Matrix\n",
    "            conf_matrix = confusion_matrix(y_true=Y_test, y_pred=Y_pred)\n",
    "            # Explainable ML\n",
    "            impurity = grid.best_estimator_.feature_importances_\n",
    "            std = np.std([tree.feature_importances_ for tree in grid.best_estimator_.estimators_], axis=0)\n",
    "            explainer = shap.TreeExplainer(grid.best_estimator_)\n",
    "            shap_values = explainer.shap_values(X_test)\n",
    "    else:\n",
    "        if authentication:\n",
    "            score_trains_c = []\n",
    "            accuracy_tests_c = []\n",
    "            precision_tests_c = []\n",
    "            recall_tests_c = []\n",
    "            f1_tests_c = []\n",
    "            score_trains_d = []\n",
    "            accuracy_tests_d = []\n",
    "            precision_tests_d = []\n",
    "            recall_tests_d = []\n",
    "            f1_tests_d = []\n",
    "            fars_tests_c = []\n",
    "            fars_tests_d = []\n",
    "            frrs_tests_c = []\n",
    "            frrs_tests_d = []\n",
    "\n",
    "            for i, (X_train_c, X_test_c, Y_train_c, Y_test_c, balance_c) in enumerate(zip(X_trains_c, X_tests_c, Y_trains_c, Y_tests_c, all_balances_c)):\n",
    "                grid_c = GridSearchCV(clf, param, n_jobs=-1, verbose=0)\n",
    "                print(f'[🤖 MODEL] {name} Charge ({i+1}/{len(X_trains_c)})', end='\\r')\n",
    "                grid_c.fit(X_train_c, Y_train_c)\n",
    "                score_trains_c.append(grid_c.best_estimator_.score(X_train_c, Y_train_c))\n",
    "                # Test scores\n",
    "                start_pred_c = time.time()\n",
    "                Y_pred_c = grid_c.best_estimator_.predict(X_test_c)\n",
    "                end_pred_c = time.time()\n",
    "\n",
    "                pred_time_c = end_pred_c - start_pred_c\n",
    "                size_c = sys.getsizeof(pickle.dumps(grid_c.best_estimator_))\n",
    "\n",
    "                complexity_c.append({\n",
    "                    'Model': name,\n",
    "                    'Prediction_Time': pred_time_c,\n",
    "                    'Size': size_c\n",
    "                })\n",
    "\n",
    "                accuracy_tests_c.append(accuracy_score(Y_test_c, Y_pred_c))\n",
    "                precision_tests_c.append(precision_score(Y_test_c, Y_pred_c))\n",
    "                recall_tests_c.append(recall_score(Y_test_c, Y_pred_c))\n",
    "                f1_tests_c.append(f1_score(Y_test_c, Y_pred_c))\n",
    "                # FARs and FRRs\n",
    "                cm_c = confusion_matrix(y_true=Y_test_c, y_pred=Y_pred_c)\n",
    "                tp_c = cm_c[0][0]\n",
    "                tn_c = cm_c[1][1]\n",
    "                fp_c = cm_c[0][1]\n",
    "                fn_c = cm_c[1][0]\n",
    "                far_c = fp_c/(fp_c+tn_c)\n",
    "                if math.isnan(far_c):\n",
    "                    far_c = 0\n",
    "                frr_c = fn_c/(fn_c+tp_c)\n",
    "                if math.isnan(frr_c):\n",
    "                    frr_c = 0\n",
    "                fars_tests_c.append(far_c)\n",
    "                frrs_tests_c.append(frr_c)\n",
    "\n",
    "                # Save results by doing the mean on each balance value\n",
    "                if (i+1) % len(np.unique(labels_c)) == 0:\n",
    "                    print()\n",
    "                    print(f'\\t[⚖️ BALANCE] {balance_c[0]}/{balance_c[1]}')\n",
    "                    print(f'\\t\\t[💪 TRAIN-C]\\t{round(np.mean(score_trains_c), 3)}')\n",
    "                    print(f'\\t\\t[📊 ACCURACY-C]\\t{round(np.mean(accuracy_tests_c), 3)}')\n",
    "                    print(f'\\t\\t[📊 PRECISION-C]\\t{round(np.mean(precision_tests_c), 3)}')\n",
    "                    print(f'\\t\\t[📊 RECALL-C]\\t{round(np.mean(recall_tests_c), 3)}')\n",
    "                    print(f'\\t\\t[📊 F1 SCORE-C]\\t{round(np.mean(f1_tests_c), 3)}')\n",
    "                    print(f'\\t\\t[🔐 FAR-C]\\t{round(np.mean(fars_tests_c), 3)}')\n",
    "                    print(f'\\t\\t[🔐 FRR-C]\\t{round(np.mean(frrs_tests_c), 3)}\\n')\n",
    "                    # Saving results\n",
    "                    train_scores_c.append(round(np.mean(score_trains_c), 3))\n",
    "                    accuracy_scores_c.append(round(np.mean(accuracy_tests_c), 3))\n",
    "                    precision_scores_c.append(round(np.mean(precision_tests_c), 3))\n",
    "                    recall_scores_c.append(round(np.mean(recall_tests_c), 3))\n",
    "                    f1_scores_c.append(round(np.mean(f1_tests_c), 3))\n",
    "                    fars_c.append(round(np.mean(fars_tests_c), 3))\n",
    "                    frrs_c.append(round(np.mean(frrs_tests_c), 3))\n",
    "                    best_params_c.append(grid_c.best_params_)\n",
    "                    model_balances_c.append(balance_c)\n",
    "                    # Resetting results\n",
    "                    score_trains_c = []\n",
    "                    accuracy_tests_c = []\n",
    "                    precision_tests_c = []\n",
    "                    recall_tests_c = []\n",
    "                    f1_tests_c = []\n",
    "                    fars_tests_c = []\n",
    "                    frrs_tests_c = []\n",
    "\n",
    "            for i, (X_train_d, X_test_d, Y_train_d, Y_test_d, balance_d) in enumerate(zip(X_trains_d, X_tests_d, Y_trains_d, Y_tests_d, all_balances_d)):\n",
    "                grid_d = GridSearchCV(clf, param, n_jobs=-1, verbose=0)\n",
    "                print(f'[🤖 MODEL] {name} Discharge ({i+1}/{len(X_trains_d)})', end='\\r')\n",
    "                grid_d.fit(X_train_d, Y_train_d)\n",
    "                score_trains_d.append(grid_d.best_estimator_.score(X_train_d, Y_train_d))\n",
    "                # Test scores\n",
    "                start_pred_d = time.time()\n",
    "                Y_pred_d = grid_d.best_estimator_.predict(X_test_d)\n",
    "                end_pred_d = time.time()\n",
    "\n",
    "                pred_time_d = end_pred_d - start_pred_d\n",
    "                size_d = sys.getsizeof(pickle.dumps(grid_d.best_estimator_))\n",
    "\n",
    "                complexity_d.append({\n",
    "                    'Model': name,\n",
    "                    'Prediction_Time': pred_time_d,\n",
    "                    'Size': size_d\n",
    "                })\n",
    "                accuracy_tests_d.append(accuracy_score(Y_test_d, Y_pred_d))\n",
    "                precision_tests_d.append(precision_score(Y_test_d, Y_pred_d))\n",
    "                recall_tests_d.append(recall_score(Y_test_d, Y_pred_d))\n",
    "                f1_tests_d.append(f1_score(Y_test_d, Y_pred_d))\n",
    "                # FARs and FRRs\n",
    "                cm_d = confusion_matrix(y_true=Y_test_d, y_pred=Y_pred_d)\n",
    "                tp_d = cm_d[0][0]\n",
    "                tn_d = cm_d[1][1]\n",
    "                fp_d = cm_d[0][1]\n",
    "                fn_d = cm_d[1][0]\n",
    "                far_d = fp_d/(fp_d+tn_d)\n",
    "                if math.isnan(far_d):\n",
    "                    far_d = 0\n",
    "                frr_d = fn_d/(fn_d+tp_d)\n",
    "                if math.isnan(frr_d):\n",
    "                    frr_d = 0\n",
    "                fars_tests_d.append(far_d)\n",
    "                frrs_tests_d.append(frr_d)\n",
    "\n",
    "                # Save results by doing the mean on each balance value\n",
    "                if (i+1) % len(np.unique(labels_d)) == 0:\n",
    "                    print()\n",
    "                    print(f'\\t[⚖️ BALANCE] {balance_d[0]}/{balance_d[1]}')\n",
    "                    print(f'\\t\\t[💪 TRAIN-D]\\t{round(np.mean(score_trains_d), 3)}')\n",
    "                    print(f'\\t\\t[📊 ACCURACY-D]\\t{round(np.mean(accuracy_tests_d), 3)}')\n",
    "                    print(f'\\t\\t[📊 PRECISION-D]\\t{round(np.mean(precision_tests_d), 3)}')\n",
    "                    print(f'\\t\\t[📊 RECALL-D]\\t{round(np.mean(recall_tests_d), 3)}')\n",
    "                    print(f'\\t\\t[📊 F1 SCORE-D]\\t{round(np.mean(f1_tests_d), 3)}')\n",
    "                    print(f'\\t\\t[🔐 FAR-D]\\t{round(np.mean(fars_tests_d), 3)}')\n",
    "                    print(f'\\t\\t[🔐 FRR-D]\\t{round(np.mean(frrs_tests_d), 3)}\\n')\n",
    "                    # Saving results\n",
    "                    train_scores_d.append(round(np.mean(score_trains_d), 3))\n",
    "                    accuracy_scores_d.append(round(np.mean(accuracy_tests_d), 3))\n",
    "                    precision_scores_d.append(round(np.mean(precision_tests_d), 3))\n",
    "                    recall_scores_d.append(round(np.mean(recall_tests_d), 3))\n",
    "                    f1_scores_d.append(round(np.mean(f1_tests_d), 3))\n",
    "                    fars_d.append(round(np.mean(fars_tests_d), 3))\n",
    "                    frrs_d.append(round(np.mean(frrs_tests_d), 3))\n",
    "                    best_params_d.append(grid_d.best_params_)\n",
    "                    model_balances_d.append(balance_d)\n",
    "                    # Resetting results\n",
    "                    score_trains_d = []\n",
    "                    accuracy_tests_d = []\n",
    "                    precision_tests_d = []\n",
    "                    recall_tests_d = []\n",
    "                    f1_tests_d = []\n",
    "                    fars_tests_d = []\n",
    "                    frrs_tests_d = []\n",
    "        else:\n",
    "            grid_c = GridSearchCV(clf, param, n_jobs=-1, verbose=0)\n",
    "            grid_d = GridSearchCV(clf, param, n_jobs=-1, verbose=0)\n",
    "            \n",
    "            print(f'[🤖 MODEL] {name}')\n",
    "\n",
    "            grid_c.fit(X_train_c, Y_train_c)\n",
    "            grid_d.fit(X_train_d, Y_train_d)\n",
    "\n",
    "            score_train_c = grid_c.best_estimator_.score(X_train_c, Y_train_c)\n",
    "            score_train_d = grid_d.best_estimator_.score(X_train_d, Y_train_d)\n",
    "            print(f'\\t[💪 TRAIN-C]\\t{round(score_train_c, 3)}')\n",
    "            print(f'\\t[💪 TRAIN-D]\\t{round(score_train_d, 3)}')\n",
    "\n",
    "            # Test scores\n",
    "            start_pred_c = time.time()\n",
    "            Y_pred_c = grid_c.best_estimator_.predict(X_test_c)\n",
    "            end_pred_c = time.time()\n",
    "\n",
    "            pred_time_c = end_pred_c - start_pred_c\n",
    "            size_c = sys.getsizeof(pickle.dumps(grid_c.best_estimator_))\n",
    "\n",
    "            complexity_c.append({\n",
    "                'Model': name,\n",
    "                'Prediction_Time': pred_time_c,\n",
    "                'Size': size_c\n",
    "            })\n",
    "            accuracy_c = accuracy_score(Y_test_c, Y_pred_c)\n",
    "            precision_c = precision_score(Y_test_c, Y_pred_c, average='macro')\n",
    "            recall_c = recall_score(Y_test_c, Y_pred_c, average='macro')\n",
    "            f1_c = f1_score(Y_test_c, Y_pred_c, average='macro')\n",
    "\n",
    "            # Test scores\n",
    "            start_pred_d = time.time()\n",
    "            Y_pred_d = grid_d.best_estimator_.predict(X_test_d)\n",
    "            end_pred_d = time.time()\n",
    "\n",
    "            pred_time_d = end_pred_d - start_pred_d\n",
    "            size_d = sys.getsizeof(pickle.dumps(grid_d.best_estimator_))\n",
    "\n",
    "            complexity_d.append({\n",
    "                'Model': name,\n",
    "                'Prediction_Time': pred_time_d,\n",
    "                'Size': size_d\n",
    "            })\n",
    "            accuracy_d = accuracy_score(Y_test_d, Y_pred_d)\n",
    "            precision_d = precision_score(Y_test_d, Y_pred_d, average='macro')\n",
    "            recall_d = recall_score(Y_test_d, Y_pred_d, average='macro')\n",
    "            f1_d = f1_score(Y_test_d, Y_pred_d, average='macro')\n",
    "\n",
    "            print(f'\\t[📊 ACCURACY-C]\\t{round(accuracy_c, 3)}')\n",
    "            print(f'\\t[📊 ACCURACY-D]\\t{round(accuracy_d, 3)}')\n",
    "            print(f'\\t[📊 PRECISION-C]\\t{round(precision_c, 3)}')\n",
    "            print(f'\\t[📊 PRECISION-D]\\t{round(precision_d, 3)}')\n",
    "            print(f'\\t[📊 RECALL-C]\\t{round(recall_c, 3)}')\n",
    "            print(f'\\t[📊 RECALL-D]\\t{round(recall_d, 3)}')\n",
    "            print(f'\\t[📊 F1 SCORE-C]\\t{round(f1_c, 3)}')\n",
    "            print(f'\\t[📊 F1 SCORE-D]\\t{round(f1_d, 3)}\\n')\n",
    "\n",
    "            train_scores_c.append(round(score_train_c, 3))\n",
    "            train_scores_d.append(round(score_train_d, 3))\n",
    "            accuracy_scores_c.append(round(accuracy_c, 3))\n",
    "            precision_scores_c.append(round(precision_c, 3))\n",
    "            recall_scores_c.append(round(recall_c, 3))\n",
    "            f1_scores_c.append(round(f1_c, 3))\n",
    "            accuracy_scores_d.append(round(accuracy_d, 3))\n",
    "            precision_scores_d.append(round(precision_d, 3))\n",
    "            recall_scores_d.append(round(recall_d, 3))\n",
    "            f1_scores_d.append(round(f1_d, 3))\n",
    "            best_params_c.append(grid_c.best_params_)\n",
    "            best_params_d.append(grid_d.best_params_)\n",
    "\n",
    "        # Feature importance for Random Forest\n",
    "        if name == 'Random Forest':\n",
    "            # Confusion Matrix\n",
    "            conf_matrix_c = confusion_matrix(y_true=Y_test_c, y_pred=Y_pred_c)\n",
    "            conf_matrix_d = confusion_matrix(y_true=Y_test_d, y_pred=Y_pred_d)\n",
    "            # Explainable ML\n",
    "            impurity_c = grid_c.best_estimator_.feature_importances_\n",
    "            impurity_d = grid_d.best_estimator_.feature_importances_\n",
    "            std_c = np.std([tree.feature_importances_ for tree in grid_c.best_estimator_.estimators_], axis=0)\n",
    "            std_d = np.std([tree.feature_importances_ for tree in grid_d.best_estimator_.estimators_], axis=0)\n",
    "            explainer_c = shap.TreeExplainer(grid_c.best_estimator_)\n",
    "            explainer_d = shap.TreeExplainer(grid_d.best_estimator_)\n",
    "            shap_values_c = explainer_c.shap_values(X_test_c)\n",
    "            shap_values_d = explainer_d.shap_values(X_test_d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not separate:\n",
    "    fig, ax = plt.subplots(figsize=(5, 5))\n",
    "    ax.matshow(conf_matrix, cmap=plt.cm.Oranges, alpha=0.3)\n",
    "    for i in range(conf_matrix.shape[0]):\n",
    "        for j in range(conf_matrix.shape[1]):\n",
    "            ax.text(x=j, y=i, s=conf_matrix[i, j],\n",
    "                    va='center', ha='center', size='xx-large')\n",
    "\n",
    "    plt.xlabel('Predictions', fontsize=18)\n",
    "    plt.ylabel('Actuals', fontsize=18)\n",
    "    plt.title('Confusion Matrix for Random Forest', fontsize=18)\n",
    "    \n",
    "    saveConfPath = os.path.join(imageFolder, saveBase)\n",
    "    saveConfPath += '_confusionMatrix.pdf'\n",
    "    plt.savefig(saveConfPath)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 🔝 Feature Importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not separate:\n",
    "    forest_impurity = pd.Series(impurity, index=X_train.columns).nlargest(20)\n",
    "\n",
    "    fig, ax = plt.subplots()\n",
    "    forest_impurity.plot.bar(ax=ax)  # , yerr=std)\n",
    "    ax.set_title(\"Feature importances using MDI\")\n",
    "    ax.set_ylabel(\"Mean decrease in impurity\")\n",
    "else:\n",
    "    forest_impurity_c = pd.Series(impurity_c, index=X_train_c.columns).nlargest(20)\n",
    "    forest_impurity_d = pd.Series(impurity_d, index=X_train_d.columns).nlargest(20)\n",
    "\n",
    "    fig, axs = plt.subplots(1, 2, figsize=(10, 5))\n",
    "    axs = axs.ravel()\n",
    "\n",
    "    forest_impurity_c.plot.bar(ax=axs[0])  # , yerr=std_c)\n",
    "    forest_impurity_d.plot.bar(ax=axs[1])  # , yerr=std_d)\n",
    "    axs[0].set_title(\"Feature importances using MDI on Charge\")\n",
    "    axs[0].set_ylabel(\"Mean decrease in impurity\")\n",
    "    axs[1].set_title(\"Feature importances using MDI on Discharge\")\n",
    "    axs[1].set_ylabel(\"Mean decrease in impurity\")\n",
    "\n",
    "fig.tight_layout()\n",
    "saveMDIPath = os.path.join(imageFolder, saveBase)\n",
    "saveMDIPath += '_MDI.pdf'\n",
    "plt.savefig(saveMDIPath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not separate:\n",
    "    shap.summary_plot(shap_values, X_test, plot_type=\"bar\", show=False)\n",
    "    saveShapPath = os.path.join(imageFolder, saveBase)\n",
    "    saveShapPath += '_Shap.pdf'\n",
    "    plt.savefig(saveShapPath)\n",
    "else:\n",
    "    shap.summary_plot(shap_values_c, X_test_c, plot_type=\"bar\", show=False)\n",
    "    saveShapPath = os.path.join(imageFolder, saveBase)\n",
    "    saveShapPath += '_ShapCharge.pdf'\n",
    "    plt.savefig(saveShapPath)\n",
    "\n",
    "    shap.summary_plot(shap_values_d, X_test_d, plot_type=\"bar\", show=False)\n",
    "    saveShapPath = os.path.join(imageFolder, saveBase)\n",
    "    saveShapPath += '_ShapDischarge.pdf'\n",
    "    plt.savefig(saveShapPath)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 💾 Saving Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not separate:\n",
    "    df_list = []\n",
    "\n",
    "    name_idx = 0\n",
    "    for i, (acc, prec, rec, f1) in enumerate(zip(accuracy_scores, precision_scores, recall_scores, f1_scores)):\n",
    "        df_list.append({\n",
    "            'Model': names[name_idx],\n",
    "            'ID': id[-1],\n",
    "            'Authentication': authentication,\n",
    "            'Accuracy': acc,\n",
    "            'Precision': prec,\n",
    "            'Recall': rec,\n",
    "            'F1': f1\n",
    "        })\n",
    "        if authentication:\n",
    "            if (i+1) % len(BALANCES) == 0:\n",
    "                name_idx += 1\n",
    "        else:\n",
    "            name_idx += 1\n",
    "\n",
    "    if authentication:\n",
    "        for i, (far, frr, dfl) in enumerate(zip(fars, frrs, df_list)):\n",
    "            dfl['FAR'] = far\n",
    "            dfl['FRR'] = frr\n",
    "            dfl['Balance'] = str(model_balances[i][0]) + '/' + str(model_balances[i][1])\n",
    "\n",
    "    df_csv = pd.DataFrame(df_list)\n",
    "    savePath = os.path.join(RESULTS, saveBase)\n",
    "    savePath += '.csv'\n",
    "    df_csv.to_csv(savePath)\n",
    "\n",
    "    # Saving times and sizes\n",
    "    df_comp = pd.DataFrame(complexity)\n",
    "    timeSavePath = os.path.join(COMPLEXITY, saveBase)\n",
    "    timeSavePath += '.csv'\n",
    "    df_comp.to_csv(timeSavePath)\n",
    "else:\n",
    "    df_list_c = []\n",
    "    df_list_d = []\n",
    "\n",
    "    name_idx = 0\n",
    "    for i, (acc_c, prec_c, rec_c, f1_c) in enumerate(zip(accuracy_scores_c, precision_scores_c, recall_scores_c, f1_scores_c)):\n",
    "        df_list_c.append({\n",
    "            'Model': names[name_idx],\n",
    "            'ID': id[-1],\n",
    "            'Authentication': authentication,\n",
    "            'Accuracy': acc_c,\n",
    "            'Precision': prec_c,\n",
    "            'Recall': rec_c,\n",
    "            'F1': f1_c\n",
    "        })\n",
    "        if authentication:\n",
    "            if (i+1) % len(BALANCES) == 0:\n",
    "                name_idx += 1\n",
    "        else:\n",
    "            name_idx += 1\n",
    "\n",
    "    if authentication:\n",
    "        for i, (far_c, frr_c, dfl_c) in enumerate(zip(fars_c, frrs_c, df_list_c)):\n",
    "            dfl_c['FAR'] = far_c\n",
    "            dfl_c['FRR'] = frr_c\n",
    "            dfl_c['Balance'] = str(model_balances_c[i][0]) + '/' + str(model_balances_c[i][1])\n",
    "\n",
    "    name_idx = 0\n",
    "    for i, (acc_d, prec_d, rec_d, f1_d) in enumerate(zip(accuracy_scores_d, precision_scores_d, recall_scores_d, f1_scores_d)):\n",
    "        df_list_d.append({\n",
    "            'Model': names[name_idx],\n",
    "            'ID': id[-1],\n",
    "            'Authentication': authentication,\n",
    "            'Accuracy': acc_d,\n",
    "            'Precision': prec_d,\n",
    "            'Recall': rec_d,\n",
    "            'F1': f1_d\n",
    "        })\n",
    "        if authentication:\n",
    "            if (i+1) % len(BALANCES) == 0:\n",
    "                name_idx += 1\n",
    "        else:\n",
    "            name_idx += 1\n",
    "\n",
    "    if authentication:\n",
    "        for i, (far_d, frr_d, dfl_d) in enumerate(zip(fars_d, frrs_d, df_list_d)):\n",
    "            dfl_d['FAR'] = far_d\n",
    "            dfl_d['FRR'] = frr_d\n",
    "            dfl_d['Balance'] = str(model_balances_d[i][0]) + '/' + str(model_balances_d[i][1])\n",
    "\n",
    "    df_csv_c = pd.DataFrame(df_list_c)\n",
    "    df_csv_d = pd.DataFrame(df_list_d)\n",
    "    savePath_c = os.path.join(RESULTS, saveBase)\n",
    "    savePath_c += '_Charge.csv'\n",
    "    df_csv_c.to_csv(savePath_c)\n",
    "    savePath_d = os.path.join(RESULTS, saveBase)\n",
    "    savePath_d += '_Discharge.csv'\n",
    "    df_csv_d.to_csv(savePath_d)\n",
    "\n",
    "    # Saving times and sizes\n",
    "    df_comp_c = pd.DataFrame(complexity_c)\n",
    "    timeSavePath_c = os.path.join(COMPLEXITY, saveBase)\n",
    "    timeSavePath_c += '_Charge.csv'\n",
    "    df_comp_c.to_csv(timeSavePath_c)\n",
    "\n",
    "    df_comp_d = pd.DataFrame(complexity_d)\n",
    "    timeSavePath_d = os.path.join(COMPLEXITY, saveBase)\n",
    "    timeSavePath_d += '_Disharge.csv'\n",
    "    df_comp_d.to_csv(timeSavePath_d)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}