meta_estimator_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import joblib\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from utils.utils import load_data, remove_zero_features, load_confounders, standardize, label_freq_sorted, pca_transform\n",
    "from utils.utils import generate_oversampled_set, generate_undersampled_set, generate_label_stats\n",
    "from utils.utils import compute_scores, compute_auprc_auroc_scores_moc\n",
    "\n",
    "from sklearn.dummy import DummyClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import HistGradientBoostingClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.multioutput import MultiOutputClassifier, ClassifierChain\n",
    "\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of physical cores: 8\n"
     ]
    }
   ],
   "source": [
    "N_CORES = joblib.cpu_count(only_physical_cores=True)\n",
    "print(f\"Number of physical cores: {N_CORES}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load data for classification task\n",
    "subject_data, features, diagnoses = load_data('classification')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove zero features\n",
    "F = remove_zero_features(features.iloc[:,1:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load confounders\n",
    "C = load_confounders(subject_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of samples: 2815\n",
      "Number of features: 922\n"
     ]
    }
   ],
   "source": [
    "# Standardize\n",
    "X = standardize(F)\n",
    "print(f\"Number of samples: {X.shape[0]}\")\n",
    "print(f\"Number of features: {X.shape[1]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of labels: 13\n"
     ]
    }
   ],
   "source": [
    "# Remove ID column\n",
    "Y = diagnoses.iloc[:,1:]\n",
    "print(f\"Number of labels: {Y.shape[1]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "boot_iter = 100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Use dataset with original label distribution (no resampling)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean imbalance ratio: 7.995236107963529\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Absolute frequency</th>\n",
       "      <th>Relative frequency</th>\n",
       "      <th>Imbalance ratio</th>\n",
       "      <th>Within-label imbalance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Trauma_And_Stress_RelatedDisorders</th>\n",
       "      <td>152</td>\n",
       "      <td>0.053996</td>\n",
       "      <td>11.519737</td>\n",
       "      <td>0.053996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DepressiveDisorders</th>\n",
       "      <td>300</td>\n",
       "      <td>0.106572</td>\n",
       "      <td>5.836667</td>\n",
       "      <td>0.106572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Attention_Deficit_HyperactivityDisorder</th>\n",
       "      <td>1751</td>\n",
       "      <td>0.622025</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.622025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MotorDisorder</th>\n",
       "      <td>229</td>\n",
       "      <td>0.081350</td>\n",
       "      <td>7.646288</td>\n",
       "      <td>0.081350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AutismSpectrumDisorder</th>\n",
       "      <td>398</td>\n",
       "      <td>0.141385</td>\n",
       "      <td>4.399497</td>\n",
       "      <td>0.141385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CommunicationDisorder</th>\n",
       "      <td>493</td>\n",
       "      <td>0.175133</td>\n",
       "      <td>3.551724</td>\n",
       "      <td>0.175133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>OtherDisorders</th>\n",
       "      <td>135</td>\n",
       "      <td>0.047957</td>\n",
       "      <td>12.970370</td>\n",
       "      <td>0.047957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SpecificLearningDisorder</th>\n",
       "      <td>761</td>\n",
       "      <td>0.270337</td>\n",
       "      <td>2.300920</td>\n",
       "      <td>0.270337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Obsessive_Compulsive_And_RelatedDisorders</th>\n",
       "      <td>137</td>\n",
       "      <td>0.048668</td>\n",
       "      <td>12.781022</td>\n",
       "      <td>0.048668</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Disruptive</th>\n",
       "      <td>436</td>\n",
       "      <td>0.154885</td>\n",
       "      <td>4.016055</td>\n",
       "      <td>0.154885</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IntellectualDisability</th>\n",
       "      <td>59</td>\n",
       "      <td>0.020959</td>\n",
       "      <td>29.677966</td>\n",
       "      <td>0.020959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EliminationDisorder</th>\n",
       "      <td>269</td>\n",
       "      <td>0.095560</td>\n",
       "      <td>6.509294</td>\n",
       "      <td>0.095560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AnxietyDisorders</th>\n",
       "      <td>1013</td>\n",
       "      <td>0.359858</td>\n",
       "      <td>1.728529</td>\n",
       "      <td>0.359858</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           Absolute frequency  \\\n",
       "Trauma_And_Stress_RelatedDisorders                        152   \n",
       "DepressiveDisorders                                       300   \n",
       "Attention_Deficit_HyperactivityDisorder                  1751   \n",
       "MotorDisorder                                             229   \n",
       "AutismSpectrumDisorder                                    398   \n",
       "CommunicationDisorder                                     493   \n",
       "OtherDisorders                                            135   \n",
       "SpecificLearningDisorder                                  761   \n",
       "Obsessive_Compulsive_And_RelatedDisorders                 137   \n",
       "Disruptive                                                436   \n",
       "IntellectualDisability                                     59   \n",
       "EliminationDisorder                                       269   \n",
       "AnxietyDisorders                                         1013   \n",
       "\n",
       "                                           Relative frequency  \\\n",
       "Trauma_And_Stress_RelatedDisorders                   0.053996   \n",
       "DepressiveDisorders                                  0.106572   \n",
       "Attention_Deficit_HyperactivityDisorder              0.622025   \n",
       "MotorDisorder                                        0.081350   \n",
       "AutismSpectrumDisorder                               0.141385   \n",
       "CommunicationDisorder                                0.175133   \n",
       "OtherDisorders                                       0.047957   \n",
       "SpecificLearningDisorder                             0.270337   \n",
       "Obsessive_Compulsive_And_RelatedDisorders            0.048668   \n",
       "Disruptive                                           0.154885   \n",
       "IntellectualDisability                               0.020959   \n",
       "EliminationDisorder                                  0.095560   \n",
       "AnxietyDisorders                                     0.359858   \n",
       "\n",
       "                                           Imbalance ratio  \\\n",
       "Trauma_And_Stress_RelatedDisorders               11.519737   \n",
       "DepressiveDisorders                               5.836667   \n",
       "Attention_Deficit_HyperactivityDisorder           1.000000   \n",
       "MotorDisorder                                     7.646288   \n",
       "AutismSpectrumDisorder                            4.399497   \n",
       "CommunicationDisorder                             3.551724   \n",
       "OtherDisorders                                   12.970370   \n",
       "SpecificLearningDisorder                          2.300920   \n",
       "Obsessive_Compulsive_And_RelatedDisorders        12.781022   \n",
       "Disruptive                                        4.016055   \n",
       "IntellectualDisability                           29.677966   \n",
       "EliminationDisorder                               6.509294   \n",
       "AnxietyDisorders                                  1.728529   \n",
       "\n",
       "                                           Within-label imbalance  \n",
       "Trauma_And_Stress_RelatedDisorders                       0.053996  \n",
       "DepressiveDisorders                                      0.106572  \n",
       "Attention_Deficit_HyperactivityDisorder                  0.622025  \n",
       "MotorDisorder                                            0.081350  \n",
       "AutismSpectrumDisorder                                   0.141385  \n",
       "CommunicationDisorder                                    0.175133  \n",
       "OtherDisorders                                           0.047957  \n",
       "SpecificLearningDisorder                                 0.270337  \n",
       "Obsessive_Compulsive_And_RelatedDisorders                0.048668  \n",
       "Disruptive                                               0.154885  \n",
       "IntellectualDisability                                   0.020959  \n",
       "EliminationDisorder                                      0.095560  \n",
       "AnxietyDisorders                                         0.359858  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_stats, mean_ir = generate_label_stats(Y, True)\n",
    "print(f\"Mean imbalance ratio: {mean_ir}\")\n",
    "label_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split dataset into train and test (holdout) set\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.25, random_state=0)\n",
    "\n",
    "# Confounder \n",
    "C_train, C_test = C.loc[X_train.index], C.loc[X_test.index]\n",
    "\n",
    "# PCA features\n",
    "X_pca, stats = pca_transform(F, stats=True)\n",
    "X_pca_train, X_pca_test = X_pca.loc[X_train.index], X_pca.loc[X_test.index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of samples in training set: 2111\n",
      "Number of samples in test set: 704\n"
     ]
    }
   ],
   "source": [
    "print(f\"Number of samples in training set: {len(X_train)}\")\n",
    "print(f\"Number of samples in test set: {len(X_test)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.1. MultiOutputClassifier\n",
    "Evaluate classification models wrapped in meta estimator MultiOutputClassifier with respect to multi-label performance metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1.1. Dummy estimators"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.1.1.1. Always zero baseline estimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean scores with SE and 95% confidence intervals:\n",
      "\n",
      "auprc_macro:                  0.17 (0.00) [0.16, 0.17]\n",
      "auprc_weighted:               0.31 (0.01) [0.30, 0.32]\n",
      "auroc_macro:                  0.50 (0.00) [0.50, 0.50]\n",
      "auroc_weighted:               0.50 (0.00) [0.50, 0.50]\n",
      "brier_macro:                  0.17 (0.00) [0.16, 0.17]\n",
      "brier_weighted:               0.05 (0.00) [0.05, 0.05]\n",
      "balanced_accuracy_macro:      0.50 (0.00) [0.50, 0.50]\n",
      "balanced_accuracy_weighted:   0.08 (0.00) [0.08, 0.08]\n",
      "f1_micro:                     0.00 (0.00) [0.00, 0.00]\n",
      "hamming:                      0.17 (0.00) [0.16, 0.17]\n",
      "subset_accuracy:              0.10 (0.01) [0.08, 0.12]\n"
     ]
    }
   ],
   "source": [
    "clf = DummyClassifier(strategy='constant', constant=0 ,random_state=0)\n",
    "meta_clf = MultiOutputClassifier(clf).fit(X_train, Y_train)\n",
    "compute_scores(meta_clf, X_test, Y_test, boot_iter)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.1.1.2. Label proportion baseline estimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean scores with SE and 95% confidence intervals:\n",
      "\n",
      "auprc_macro:                  0.17 (0.00) [0.16, 0.17]\n",
      "auprc_weighted:               0.31 (0.01) [0.30, 0.32]\n",
      "auroc_macro:                  0.50 (0.00) [0.50, 0.50]\n",
      "auroc_weighted:               0.50 (0.00) [0.50, 0.50]\n",
      "brier_macro:                  0.12 (0.00) [0.11, 0.12]\n",
      "brier_weighted:               0.03 (0.00) [0.03, 0.03]\n",
      "balanced_accuracy_macro:      0.50 (0.00) [0.50, 0.50]\n",
      "balanced_accuracy_weighted:   0.08 (0.00) [0.08, 0.08]\n",
      "f1_micro:                     0.38 (0.01) [0.36, 0.40]\n",
      "hamming:                      0.15 (0.00) [0.14, 0.16]\n",
      "subset_accuracy:              0.10 (0.01) [0.08, 0.12]\n"
     ]
    }
   ],
   "source": [
    "clf = DummyClassifier(strategy='prior', random_state=0)\n",
    "meta_clf = MultiOutputClassifier(clf).fit(X_train, Y_train)\n",
    "compute_scores(meta_clf, X_test, Y_test, boot_iter)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1.2. Confounders as features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.1.2.1. Logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean scores with SE and 95% confidence intervals:\n",
      "\n",
      "auprc_macro:                  0.18 (0.01) [0.17, 0.19]\n",
      "auprc_weighted:               0.32 (0.01) [0.31, 0.34]\n",
      "auroc_macro:                  0.51 (0.01) [0.49, 0.53]\n",
      "auroc_weighted:               0.51 (0.01) [0.49, 0.53]\n",
      "brier_macro:                  0.25 (0.00) [0.25, 0.25]\n",
      "brier_weighted:               0.04 (0.00) [0.04, 0.04]\n",
      "balanced_accuracy_macro:      0.51 (0.01) [0.49, 0.53]\n",
      "balanced_accuracy_weighted:   0.08 (0.00) [0.08, 0.09]\n",
      "f1_micro:                     0.25 (0.01) [0.24, 0.27]\n",
      "hamming:                      0.47 (0.00) [0.46, 0.48]\n",
      "subset_accuracy:              0.00 (0.00) [0.00, 0.00]\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(class_weight='balanced', max_iter=10000, random_state=0)\n",
    "meta_clf = MultiOutputClassifier(clf).fit(C_train, Y_train)\n",
    "compute_scores(meta_clf, C_test, Y_test, boot_iter)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1.3. PCA-projected data (top-10 components) as features "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.1.3.1. Logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean scores with SE and 95% confidence intervals:\n",
      "\n",
      "auprc_macro:                  0.23 (0.01) [0.21, 0.25]\n",
      "auprc_weighted:               0.37 (0.01) [0.35, 0.40]\n",
      "auroc_macro:                  0.60 (0.01) [0.58, 0.61]\n",
      "auroc_weighted:               0.58 (0.01) [0.56, 0.60]\n",
      "brier_macro:                  0.24 (0.00) [0.23, 0.24]\n",
      "brier_weighted:               0.04 (0.00) [0.04, 0.04]\n",
      "balanced_accuracy_macro:      0.57 (0.01) [0.55, 0.59]\n",
      "balanced_accuracy_weighted:   0.09 (0.00) [0.09, 0.10]\n",
      "f1_micro:                     0.31 (0.01) [0.30, 0.33]\n",
      "hamming:                      0.43 (0.00) [0.42, 0.44]\n",
      "subset_accuracy:              0.00 (0.00) [0.00, 0.00]\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(class_weight='balanced', max_iter=10000, random_state=0)\n",
    "meta_clf = MultiOutputClassifier(clf).fit(X_pca_train, Y_train)\n",
    "compute_scores(meta_clf, X_pca_test, Y_test, boot_iter)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1.4. Original features (standardized)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.1.4.1. Logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean scores with SE and 95% confidence intervals:\n",
      "\n",
      "auprc_macro:                  0.21 (0.01) [0.19, 0.22]\n",
      "auprc_weighted:               0.36 (0.01) [0.34, 0.38]\n",
      "auroc_macro:                  0.57 (0.01) [0.55, 0.59]\n",
      "auroc_weighted:               0.56 (0.01) [0.54, 0.58]\n",
      "brier_macro:                  0.21 (0.00) [0.21, 0.22]\n",
      "brier_weighted:               0.05 (0.00) [0.04, 0.05]\n",
      "balanced_accuracy_macro:      0.53 (0.01) [0.51, 0.54]\n",
      "balanced_accuracy_weighted:   0.09 (0.00) [0.09, 0.09]\n",
      "f1_micro:                     0.33 (0.01) [0.31, 0.35]\n",
      "hamming:                      0.27 (0.00) [0.26, 0.28]\n",
      "subset_accuracy:              0.01 (0.00) [0.01, 0.02]\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(class_weight='balanced', max_iter=10000, random_state=0)\n",
    "meta_clf = MultiOutputClassifier(clf).fit(X_train, Y_train)\n",
    "compute_scores(meta_clf, X_test, Y_test, boot_iter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean scores with SE and 95% confidence intervals:\n",
      "\n",
      "AUPRC:\n",
      "Trauma_And_Stress_RelatedDisorders:               0.07 (0.01) [0.05, 0.10]\n",
      "DepressiveDisorders:                              0.19 (0.03) [0.14, 0.26]\n",
      "Attention_Deficit_HyperactivityDisorder:          0.68 (0.03) [0.62, 0.73]\n",
      "MotorDisorder:                                    0.13 (0.03) [0.09, 0.19]\n",
      "AutismSpectrumDisorder:                           0.16 (0.02) [0.12, 0.20]\n",
      "CommunicationDisorder:                            0.22 (0.03) [0.17, 0.27]\n",
      "OtherDisorders:                                   0.11 (0.03) [0.06, 0.19]\n",
      "SpecificLearningDisorder:                         0.27 (0.02) [0.24, 0.32]\n",
      "Obsessive_Compulsive_And_RelatedDisorders:        0.07 (0.01) [0.05, 0.10]\n",
      "Disruptive:                                       0.17 (0.02) [0.14, 0.23]\n",
      "IntellectualDisability:                           0.09 (0.05) [0.03, 0.19]\n",
      "EliminationDisorder:                              0.12 (0.03) [0.08, 0.18]\n",
      "AnxietyDisorders:                                 0.42 (0.03) [0.36, 0.48]\n",
      "\n",
      "AUROC:\n",
      "Trauma_And_Stress_RelatedDisorders:               0.54 (0.04) [0.46, 0.63]\n",
      "DepressiveDisorders:                              0.67 (0.03) [0.61, 0.72]\n",
      "Attention_Deficit_HyperactivityDisorder:          0.59 (0.02) [0.54, 0.62]\n",
      "MotorDisorder:                                    0.49 (0.04) [0.41, 0.57]\n",
      "AutismSpectrumDisorder:                           0.54 (0.03) [0.48, 0.60]\n",
      "CommunicationDisorder:                            0.53 (0.03) [0.48, 0.59]\n",
      "OtherDisorders:                                   0.57 (0.05) [0.48, 0.66]\n",
      "SpecificLearningDisorder:                         0.52 (0.02) [0.48, 0.57]\n",
      "Obsessive_Compulsive_And_RelatedDisorders:        0.56 (0.04) [0.49, 0.64]\n",
      "Disruptive:                                       0.54 (0.03) [0.46, 0.60]\n",
      "IntellectualDisability:                           0.73 (0.07) [0.62, 0.86]\n",
      "EliminationDisorder:                              0.51 (0.04) [0.44, 0.57]\n",
      "AnxietyDisorders:                                 0.57 (0.02) [0.53, 0.61]\n"
     ]
    }
   ],
   "source": [
    "compute_auprc_auroc_scores_moc(meta_clf, X_test, Y_test, boot_iter)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.1.4.2. SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean scores with SE and 95% confidence intervals:\n",
      "\n",
      "auprc_macro:                  0.23 (0.01) [0.22, 0.25]\n",
      "auprc_weighted:               0.39 (0.01) [0.36, 0.41]\n",
      "auroc_macro:                  0.60 (0.01) [0.58, 0.63]\n",
      "auroc_weighted:               0.60 (0.01) [0.58, 0.63]\n",
      "brier_macro:                  0.11 (0.00) [0.11, 0.12]\n",
      "brier_weighted:               0.03 (0.00) [0.03, 0.03]\n",
      "balanced_accuracy_macro:      0.54 (0.01) [0.53, 0.55]\n",
      "balanced_accuracy_weighted:   0.09 (0.00) [0.09, 0.10]\n",
      "f1_micro:                     0.38 (0.01) [0.36, 0.40]\n",
      "hamming:                      0.23 (0.00) [0.22, 0.24]\n",
      "subset_accuracy:              0.04 (0.01) [0.03, 0.06]\n"
     ]
    }
   ],
   "source": [
    "clf = SVC(class_weight='balanced', kernel='rbf', gamma='scale', probability=True, random_state=0)\n",
    "meta_clf = MultiOutputClassifier(clf, n_jobs=-1).fit(X_train, Y_train)\n",
    "compute_scores(meta_clf, X_test, Y_test, boot_iter)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.1.4.3. Histogram-based Gradient Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean scores with SE and 95% confidence intervals:\n",
      "\n",
      "auprc_macro:                  0.23 (0.01) [0.21, 0.25]\n",
      "auprc_weighted:               0.36 (0.01) [0.34, 0.39]\n",
      "auroc_macro:                  0.59 (0.01) [0.57, 0.62]\n",
      "auroc_weighted:               0.58 (0.01) [0.55, 0.60]\n",
      "brier_macro:                  0.13 (0.00) [0.12, 0.13]\n",
      "brier_weighted:               0.03 (0.00) [0.03, 0.03]\n",
      "balanced_accuracy_macro:      0.51 (0.00) [0.50, 0.51]\n",
      "balanced_accuracy_weighted:   0.09 (0.00) [0.09, 0.09]\n",
      "f1_micro:                     0.37 (0.01) [0.36, 0.39]\n",
      "hamming:                      0.15 (0.00) [0.15, 0.16]\n",
      "subset_accuracy:              0.11 (0.01) [0.09, 0.13]\n"
     ]
    }
   ],
   "source": [
    "clf = HistGradientBoostingClassifier(random_state=0)\n",
    "meta_clf = MultiOutputClassifier(clf, n_jobs=-1).fit(X_train, Y_train)\n",
    "compute_scores(meta_clf, X_test, Y_test, boot_iter)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.1.4.4. MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean scores with SE and 95% confidence intervals:\n",
      "\n",
      "auprc_macro:                  0.22 (0.01) [0.20, 0.23]\n",
      "auprc_weighted:               0.36 (0.01) [0.34, 0.38]\n",
      "auroc_macro:                  0.58 (0.01) [0.55, 0.60]\n",
      "auroc_weighted:               0.57 (0.01) [0.55, 0.59]\n",
      "brier_macro:                  0.16 (0.00) [0.15, 0.16]\n",
      "brier_weighted:               0.04 (0.00) [0.04, 0.04]\n",
      "balanced_accuracy_macro:      0.52 (0.00) [0.52, 0.53]\n",
      "balanced_accuracy_weighted:   0.09 (0.00) [0.09, 0.09]\n",
      "f1_micro:                     0.37 (0.01) [0.35, 0.39]\n",
      "hamming:                      0.18 (0.00) [0.18, 0.19]\n",
      "subset_accuracy:              0.08 (0.01) [0.07, 0.10]\n"
     ]
    }
   ],
   "source": [
    "clf = MLPClassifier(random_state=0)\n",
    "meta_clf = MultiOutputClassifier(clf, n_jobs=-1).fit(X_train, Y_train)\n",
    "compute_scores(meta_clf, X_test, Y_test, boot_iter)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.2. ClassifierChain (ordered by frequency) \n",
    "Evaluate classification models wrapped in meta estimator ClassifierChain with respect to multi-label performance metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "by_freq = label_freq_sorted(Y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.2.1. Logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean scores with SE and 95% confidence intervals:\n",
      "\n",
      "auprc_macro:                  0.21 (0.01) [0.19, 0.22]\n",
      "auprc_weighted:               0.36 (0.01) [0.34, 0.38]\n",
      "auroc_macro:                  0.56 (0.01) [0.55, 0.59]\n",
      "auroc_weighted:               0.56 (0.01) [0.54, 0.58]\n",
      "brier_macro:                  0.21 (0.00) [0.20, 0.22]\n",
      "brier_weighted:               0.04 (0.00) [0.04, 0.05]\n",
      "balanced_accuracy_macro:      0.53 (0.01) [0.52, 0.55]\n",
      "balanced_accuracy_weighted:   0.09 (0.00) [0.09, 0.09]\n",
      "f1_micro:                     0.34 (0.01) [0.32, 0.36]\n",
      "hamming:                      0.26 (0.00) [0.26, 0.27]\n",
      "subset_accuracy:              0.03 (0.01) [0.01, 0.04]\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(class_weight='balanced', max_iter=10000, random_state=0)\n",
    "meta_clf = ClassifierChain(clf, order=by_freq, random_state=0).fit(X_train, Y_train)\n",
    "compute_scores(meta_clf, X_test, Y_test, boot_iter, chain=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.2.2. Histogram-based Gradient Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean scores with SE and 95% confidence intervals:\n",
      "\n",
      "auprc_macro:                  0.23 (0.01) [0.21, 0.26]\n",
      "auprc_weighted:               0.36 (0.01) [0.34, 0.39]\n",
      "auroc_macro:                  0.59 (0.01) [0.57, 0.61]\n",
      "auroc_weighted:               0.57 (0.01) [0.55, 0.60]\n",
      "brier_macro:                  0.13 (0.00) [0.12, 0.13]\n",
      "brier_weighted:               0.03 (0.00) [0.03, 0.03]\n",
      "balanced_accuracy_macro:      0.51 (0.00) [0.50, 0.51]\n",
      "balanced_accuracy_weighted:   0.09 (0.00) [0.09, 0.09]\n",
      "f1_micro:                     0.37 (0.01) [0.36, 0.39]\n",
      "hamming:                      0.15 (0.00) [0.15, 0.16]\n",
      "subset_accuracy:              0.11 (0.01) [0.09, 0.13]\n"
     ]
    }
   ],
   "source": [
    "clf = HistGradientBoostingClassifier(random_state=0)\n",
    "meta_clf = ClassifierChain(clf, order=by_freq, random_state=0).fit(X_train, Y_train)\n",
    "compute_scores(meta_clf, X_test, Y_test, boot_iter, chain=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.3. ClassifierChain (random order) \n",
    "Evaluate classification models wrapped in meta estimator ClassifierChain with respect to multi-label performance metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.3.1. Logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean scores with SE and 95% confidence intervals:\n",
      "\n",
      "auprc_macro:                  0.21 (0.01) [0.20, 0.22]\n",
      "auprc_weighted:               0.36 (0.01) [0.34, 0.38]\n",
      "auroc_macro:                  0.57 (0.01) [0.55, 0.59]\n",
      "auroc_weighted:               0.56 (0.01) [0.54, 0.58]\n",
      "brier_macro:                  0.21 (0.00) [0.21, 0.22]\n",
      "brier_weighted:               0.05 (0.00) [0.04, 0.05]\n",
      "balanced_accuracy_macro:      0.53 (0.01) [0.51, 0.54]\n",
      "balanced_accuracy_weighted:   0.09 (0.00) [0.09, 0.09]\n",
      "f1_micro:                     0.34 (0.01) [0.32, 0.36]\n",
      "hamming:                      0.26 (0.00) [0.26, 0.28]\n",
      "subset_accuracy:              0.02 (0.01) [0.01, 0.03]\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(class_weight='balanced', max_iter=10000, random_state=0)\n",
    "meta_clf = ClassifierChain(clf, random_state=0).fit(X_train, Y_train)\n",
    "compute_scores(meta_clf, X_test, Y_test, boot_iter, chain=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.3.2. Histogram-based Gradient Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean scores with SE and 95% confidence intervals:\n",
      "\n",
      "auprc_macro:                  0.23 (0.01) [0.21, 0.25]\n",
      "auprc_weighted:               0.36 (0.01) [0.34, 0.39]\n",
      "auroc_macro:                  0.59 (0.01) [0.57, 0.62]\n",
      "auroc_weighted:               0.57 (0.01) [0.55, 0.60]\n",
      "brier_macro:                  0.13 (0.00) [0.12, 0.13]\n",
      "brier_weighted:               0.03 (0.00) [0.03, 0.03]\n",
      "balanced_accuracy_macro:      0.50 (0.00) [0.50, 0.51]\n",
      "balanced_accuracy_weighted:   0.09 (0.00) [0.08, 0.09]\n",
      "f1_micro:                     0.35 (0.01) [0.34, 0.38]\n",
      "hamming:                      0.15 (0.00) [0.15, 0.16]\n",
      "subset_accuracy:              0.11 (0.01) [0.09, 0.14]\n"
     ]
    }
   ],
   "source": [
    "clf = HistGradientBoostingClassifier(random_state=0)\n",
    "meta_clf = ClassifierChain(clf, random_state=0).fit(X_train, Y_train)\n",
    "compute_scores(meta_clf, X_test, Y_test, boot_iter, chain=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Use undersampled dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.25, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean imbalance ratio: 2.956543202759316\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Absolute frequency</th>\n",
       "      <th>Relative frequency</th>\n",
       "      <th>Imbalance ratio</th>\n",
       "      <th>Within-label imbalance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Trauma_And_Stress_RelatedDisorders</th>\n",
       "      <td>38</td>\n",
       "      <td>0.051701</td>\n",
       "      <td>4.473684</td>\n",
       "      <td>0.051701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DepressiveDisorders</th>\n",
       "      <td>80</td>\n",
       "      <td>0.108844</td>\n",
       "      <td>2.125000</td>\n",
       "      <td>0.108844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Attention_Deficit_HyperactivityDisorder</th>\n",
       "      <td>170</td>\n",
       "      <td>0.231293</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.231293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MotorDisorder</th>\n",
       "      <td>52</td>\n",
       "      <td>0.070748</td>\n",
       "      <td>3.269231</td>\n",
       "      <td>0.070748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AutismSpectrumDisorder</th>\n",
       "      <td>101</td>\n",
       "      <td>0.137415</td>\n",
       "      <td>1.683168</td>\n",
       "      <td>0.137415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CommunicationDisorder</th>\n",
       "      <td>75</td>\n",
       "      <td>0.102041</td>\n",
       "      <td>2.266667</td>\n",
       "      <td>0.102041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>OtherDisorders</th>\n",
       "      <td>35</td>\n",
       "      <td>0.047619</td>\n",
       "      <td>4.857143</td>\n",
       "      <td>0.047619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SpecificLearningDisorder</th>\n",
       "      <td>128</td>\n",
       "      <td>0.174150</td>\n",
       "      <td>1.328125</td>\n",
       "      <td>0.174150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Obsessive_Compulsive_And_RelatedDisorders</th>\n",
       "      <td>32</td>\n",
       "      <td>0.043537</td>\n",
       "      <td>5.312500</td>\n",
       "      <td>0.043537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Disruptive</th>\n",
       "      <td>74</td>\n",
       "      <td>0.100680</td>\n",
       "      <td>2.297297</td>\n",
       "      <td>0.100680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IntellectualDisability</th>\n",
       "      <td>27</td>\n",
       "      <td>0.036735</td>\n",
       "      <td>6.296296</td>\n",
       "      <td>0.036735</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EliminationDisorder</th>\n",
       "      <td>73</td>\n",
       "      <td>0.099320</td>\n",
       "      <td>2.328767</td>\n",
       "      <td>0.099320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AnxietyDisorders</th>\n",
       "      <td>142</td>\n",
       "      <td>0.193197</td>\n",
       "      <td>1.197183</td>\n",
       "      <td>0.193197</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           Absolute frequency  \\\n",
       "Trauma_And_Stress_RelatedDisorders                         38   \n",
       "DepressiveDisorders                                        80   \n",
       "Attention_Deficit_HyperactivityDisorder                   170   \n",
       "MotorDisorder                                              52   \n",
       "AutismSpectrumDisorder                                    101   \n",
       "CommunicationDisorder                                      75   \n",
       "OtherDisorders                                             35   \n",
       "SpecificLearningDisorder                                  128   \n",
       "Obsessive_Compulsive_And_RelatedDisorders                  32   \n",
       "Disruptive                                                 74   \n",
       "IntellectualDisability                                     27   \n",
       "EliminationDisorder                                        73   \n",
       "AnxietyDisorders                                          142   \n",
       "\n",
       "                                           Relative frequency  \\\n",
       "Trauma_And_Stress_RelatedDisorders                   0.051701   \n",
       "DepressiveDisorders                                  0.108844   \n",
       "Attention_Deficit_HyperactivityDisorder              0.231293   \n",
       "MotorDisorder                                        0.070748   \n",
       "AutismSpectrumDisorder                               0.137415   \n",
       "CommunicationDisorder                                0.102041   \n",
       "OtherDisorders                                       0.047619   \n",
       "SpecificLearningDisorder                             0.174150   \n",
       "Obsessive_Compulsive_And_RelatedDisorders            0.043537   \n",
       "Disruptive                                           0.100680   \n",
       "IntellectualDisability                               0.036735   \n",
       "EliminationDisorder                                  0.099320   \n",
       "AnxietyDisorders                                     0.193197   \n",
       "\n",
       "                                           Imbalance ratio  \\\n",
       "Trauma_And_Stress_RelatedDisorders                4.473684   \n",
       "DepressiveDisorders                               2.125000   \n",
       "Attention_Deficit_HyperactivityDisorder           1.000000   \n",
       "MotorDisorder                                     3.269231   \n",
       "AutismSpectrumDisorder                            1.683168   \n",
       "CommunicationDisorder                             2.266667   \n",
       "OtherDisorders                                    4.857143   \n",
       "SpecificLearningDisorder                          1.328125   \n",
       "Obsessive_Compulsive_And_RelatedDisorders         5.312500   \n",
       "Disruptive                                        2.297297   \n",
       "IntellectualDisability                            6.296296   \n",
       "EliminationDisorder                               2.328767   \n",
       "AnxietyDisorders                                  1.197183   \n",
       "\n",
       "                                           Within-label imbalance  \n",
       "Trauma_And_Stress_RelatedDisorders                       0.051701  \n",
       "DepressiveDisorders                                      0.108844  \n",
       "Attention_Deficit_HyperactivityDisorder                  0.231293  \n",
       "MotorDisorder                                            0.070748  \n",
       "AutismSpectrumDisorder                                   0.137415  \n",
       "CommunicationDisorder                                    0.102041  \n",
       "OtherDisorders                                           0.047619  \n",
       "SpecificLearningDisorder                                 0.174150  \n",
       "Obsessive_Compulsive_And_RelatedDisorders                0.043537  \n",
       "Disruptive                                               0.100680  \n",
       "IntellectualDisability                                   0.036735  \n",
       "EliminationDisorder                                      0.099320  \n",
       "AnxietyDisorders                                         0.193197  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_under, Y_under = generate_undersampled_set(X_train, Y_train)\n",
    "label_stats, mean_ir = generate_label_stats(Y_under, True)\n",
    "print(f\"Mean imbalance ratio: {mean_ir}\")\n",
    "label_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, Y_train = X_under, Y_under"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.1. MultiOutputClassifier\n",
    "Evaluate classification models wrapped in meta estimator MultiOutputClassifier with respect to multi-label performance metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1.3. Original features (standardized)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.1.3.1. Logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean scores with SE and 95% confidence intervals:\n",
      "\n",
      "auprc_macro:                  0.20 (0.01) [0.19, 0.23]\n",
      "auprc_weighted:               0.34 (0.01) [0.32, 0.37]\n",
      "auroc_macro:                  0.55 (0.01) [0.53, 0.57]\n",
      "auroc_weighted:               0.53 (0.01) [0.51, 0.55]\n",
      "brier_macro:                  0.18 (0.00) [0.18, 0.19]\n",
      "brier_weighted:               0.05 (0.00) [0.05, 0.05]\n",
      "balanced_accuracy_macro:      0.51 (0.01) [0.50, 0.52]\n",
      "balanced_accuracy_weighted:   0.09 (0.00) [0.08, 0.09]\n",
      "f1_micro:                     0.22 (0.01) [0.20, 0.24]\n",
      "hamming:                      0.21 (0.00) [0.20, 0.22]\n",
      "subset_accuracy:              0.06 (0.01) [0.04, 0.07]\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(max_iter=10000, random_state=0)\n",
    "meta_clf = MultiOutputClassifier(clf).fit(X_train, Y_train)\n",
    "compute_scores(meta_clf, X_test, Y_test, boot_iter)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.1.3.3. Histogram-based Gradient Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean scores with SE and 95% confidence intervals:\n",
      "\n",
      "auprc_macro:                  0.23 (0.01) [0.21, 0.25]\n",
      "auprc_weighted:               0.37 (0.01) [0.35, 0.39]\n",
      "auroc_macro:                  0.57 (0.01) [0.55, 0.60]\n",
      "auroc_weighted:               0.57 (0.01) [0.55, 0.59]\n",
      "brier_macro:                  0.16 (0.00) [0.15, 0.16]\n",
      "brier_weighted:               0.05 (0.00) [0.04, 0.05]\n",
      "balanced_accuracy_macro:      0.50 (0.00) [0.50, 0.51]\n",
      "balanced_accuracy_weighted:   0.08 (0.00) [0.08, 0.08]\n",
      "f1_micro:                     0.03 (0.01) [0.02, 0.05]\n",
      "hamming:                      0.17 (0.00) [0.16, 0.18]\n",
      "subset_accuracy:              0.10 (0.01) [0.08, 0.12]\n"
     ]
    }
   ],
   "source": [
    "clf = HistGradientBoostingClassifier(random_state=0)\n",
    "meta_clf = MultiOutputClassifier(clf).fit(X_train, Y_train)\n",
    "compute_scores(meta_clf, X_test, Y_test, boot_iter)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Use oversampled dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split dataset into train and test (holdout) set\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.25, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean imbalance ratio: 1.6752910161710401\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Absolute frequency</th>\n",
       "      <th>Relative frequency</th>\n",
       "      <th>Imbalance ratio</th>\n",
       "      <th>Within-label imbalance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Trauma_And_Stress_RelatedDisorders</th>\n",
       "      <td>1895</td>\n",
       "      <td>0.112296</td>\n",
       "      <td>2.105541</td>\n",
       "      <td>0.112296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DepressiveDisorders</th>\n",
       "      <td>2577</td>\n",
       "      <td>0.152711</td>\n",
       "      <td>1.548312</td>\n",
       "      <td>0.152711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Attention_Deficit_HyperactivityDisorder</th>\n",
       "      <td>3496</td>\n",
       "      <td>0.207170</td>\n",
       "      <td>1.141304</td>\n",
       "      <td>0.207170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MotorDisorder</th>\n",
       "      <td>2386</td>\n",
       "      <td>0.141393</td>\n",
       "      <td>1.672255</td>\n",
       "      <td>0.141393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AutismSpectrumDisorder</th>\n",
       "      <td>2820</td>\n",
       "      <td>0.167111</td>\n",
       "      <td>1.414894</td>\n",
       "      <td>0.167111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CommunicationDisorder</th>\n",
       "      <td>3459</td>\n",
       "      <td>0.204978</td>\n",
       "      <td>1.153513</td>\n",
       "      <td>0.204978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>OtherDisorders</th>\n",
       "      <td>1769</td>\n",
       "      <td>0.104830</td>\n",
       "      <td>2.255512</td>\n",
       "      <td>0.104830</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SpecificLearningDisorder</th>\n",
       "      <td>3990</td>\n",
       "      <td>0.236444</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.236444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Obsessive_Compulsive_And_RelatedDisorders</th>\n",
       "      <td>2120</td>\n",
       "      <td>0.125630</td>\n",
       "      <td>1.882075</td>\n",
       "      <td>0.125630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Disruptive</th>\n",
       "      <td>2080</td>\n",
       "      <td>0.123259</td>\n",
       "      <td>1.918269</td>\n",
       "      <td>0.123259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IntellectualDisability</th>\n",
       "      <td>1644</td>\n",
       "      <td>0.097422</td>\n",
       "      <td>2.427007</td>\n",
       "      <td>0.097422</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EliminationDisorder</th>\n",
       "      <td>2335</td>\n",
       "      <td>0.138370</td>\n",
       "      <td>1.708779</td>\n",
       "      <td>0.138370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AnxietyDisorders</th>\n",
       "      <td>2572</td>\n",
       "      <td>0.152415</td>\n",
       "      <td>1.551322</td>\n",
       "      <td>0.152415</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           Absolute frequency  \\\n",
       "Trauma_And_Stress_RelatedDisorders                       1895   \n",
       "DepressiveDisorders                                      2577   \n",
       "Attention_Deficit_HyperactivityDisorder                  3496   \n",
       "MotorDisorder                                            2386   \n",
       "AutismSpectrumDisorder                                   2820   \n",
       "CommunicationDisorder                                    3459   \n",
       "OtherDisorders                                           1769   \n",
       "SpecificLearningDisorder                                 3990   \n",
       "Obsessive_Compulsive_And_RelatedDisorders                2120   \n",
       "Disruptive                                               2080   \n",
       "IntellectualDisability                                   1644   \n",
       "EliminationDisorder                                      2335   \n",
       "AnxietyDisorders                                         2572   \n",
       "\n",
       "                                           Relative frequency  \\\n",
       "Trauma_And_Stress_RelatedDisorders                   0.112296   \n",
       "DepressiveDisorders                                  0.152711   \n",
       "Attention_Deficit_HyperactivityDisorder              0.207170   \n",
       "MotorDisorder                                        0.141393   \n",
       "AutismSpectrumDisorder                               0.167111   \n",
       "CommunicationDisorder                                0.204978   \n",
       "OtherDisorders                                       0.104830   \n",
       "SpecificLearningDisorder                             0.236444   \n",
       "Obsessive_Compulsive_And_RelatedDisorders            0.125630   \n",
       "Disruptive                                           0.123259   \n",
       "IntellectualDisability                               0.097422   \n",
       "EliminationDisorder                                  0.138370   \n",
       "AnxietyDisorders                                     0.152415   \n",
       "\n",
       "                                           Imbalance ratio  \\\n",
       "Trauma_And_Stress_RelatedDisorders                2.105541   \n",
       "DepressiveDisorders                               1.548312   \n",
       "Attention_Deficit_HyperactivityDisorder           1.141304   \n",
       "MotorDisorder                                     1.672255   \n",
       "AutismSpectrumDisorder                            1.414894   \n",
       "CommunicationDisorder                             1.153513   \n",
       "OtherDisorders                                    2.255512   \n",
       "SpecificLearningDisorder                          1.000000   \n",
       "Obsessive_Compulsive_And_RelatedDisorders         1.882075   \n",
       "Disruptive                                        1.918269   \n",
       "IntellectualDisability                            2.427007   \n",
       "EliminationDisorder                               1.708779   \n",
       "AnxietyDisorders                                  1.551322   \n",
       "\n",
       "                                           Within-label imbalance  \n",
       "Trauma_And_Stress_RelatedDisorders                       0.112296  \n",
       "DepressiveDisorders                                      0.152711  \n",
       "Attention_Deficit_HyperactivityDisorder                  0.207170  \n",
       "MotorDisorder                                            0.141393  \n",
       "AutismSpectrumDisorder                                   0.167111  \n",
       "CommunicationDisorder                                    0.204978  \n",
       "OtherDisorders                                           0.104830  \n",
       "SpecificLearningDisorder                                 0.236444  \n",
       "Obsessive_Compulsive_And_RelatedDisorders                0.125630  \n",
       "Disruptive                                               0.123259  \n",
       "IntellectualDisability                                   0.097422  \n",
       "EliminationDisorder                                      0.138370  \n",
       "AnxietyDisorders                                         0.152415  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_over, Y_over = generate_oversampled_set(X_train, Y_train)\n",
    "label_stats, mean_ir = generate_label_stats(Y_over, True)\n",
    "print(f\"Mean imbalance ratio: {mean_ir}\")\n",
    "label_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, Y_train = X_over, Y_over"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.1. MultiOutputClassifier\n",
    "Evaluate classification models wrapped in meta estimator MultiOutputClassifier with respect to multi-label performance metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1.3. Original features (standardized)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.1.3.1. Logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean scores with SE and 95% confidence intervals:\n",
      "\n",
      "auprc_macro:                  0.21 (0.01) [0.19, 0.23]\n",
      "auprc_weighted:               0.35 (0.01) [0.33, 0.37]\n",
      "auroc_macro:                  0.56 (0.01) [0.54, 0.58]\n",
      "auroc_weighted:               0.56 (0.01) [0.54, 0.57]\n",
      "brier_macro:                  0.20 (0.00) [0.20, 0.21]\n",
      "brier_weighted:               0.05 (0.00) [0.05, 0.05]\n",
      "balanced_accuracy_macro:      0.52 (0.01) [0.51, 0.53]\n",
      "balanced_accuracy_weighted:   0.09 (0.00) [0.09, 0.09]\n",
      "f1_micro:                     0.33 (0.01) [0.31, 0.35]\n",
      "hamming:                      0.24 (0.00) [0.23, 0.24]\n",
      "subset_accuracy:              0.03 (0.01) [0.01, 0.04]\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(max_iter=10000, random_state=0)\n",
    "meta_clf = MultiOutputClassifier(clf, n_jobs=-1).fit(X_train, Y_train) # n_jobs=-1\n",
    "compute_scores(meta_clf, X_test, Y_test, boot_iter)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.1.3.3. Histogram-based Gradient Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean scores with SE and 95% confidence intervals:\n",
      "\n",
      "auprc_macro:                  0.23 (0.01) [0.21, 0.25]\n",
      "auprc_weighted:               0.36 (0.01) [0.33, 0.38]\n",
      "auroc_macro:                  0.59 (0.01) [0.56, 0.61]\n",
      "auroc_weighted:               0.57 (0.01) [0.55, 0.59]\n",
      "brier_macro:                  0.13 (0.00) [0.12, 0.13]\n",
      "brier_weighted:               0.03 (0.00) [0.03, 0.03]\n",
      "balanced_accuracy_macro:      0.51 (0.00) [0.50, 0.51]\n",
      "balanced_accuracy_weighted:   0.09 (0.00) [0.08, 0.09]\n",
      "f1_micro:                     0.34 (0.01) [0.32, 0.36]\n",
      "hamming:                      0.16 (0.00) [0.15, 0.16]\n",
      "subset_accuracy:              0.10 (0.01) [0.08, 0.11]\n"
     ]
    }
   ],
   "source": [
    "clf = HistGradientBoostingClassifier(random_state=0)\n",
    "meta_clf = MultiOutputClassifier(clf).fit(X_train, Y_train) # n_jobs=-1\n",
    "compute_scores(meta_clf, X_test, Y_test, boot_iter)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "neuro",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}