icr-data-augment.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9fafc4c0",
   "metadata": {
    "papermill": {
     "duration": 0.004931,
     "end_time": "2024-02-12T13:23:37.525738",
     "exception": false,
     "start_time": "2024-02-12T13:23:37.520807",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Data labeled as \"greeks\" is from the competition available only for training set, not for testing set. In this notebook, so far without significant succes, I'm trying to create an ensemble classification model that could predict the missing labels."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8774fcb6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-12T13:23:37.536281Z",
     "iopub.status.busy": "2024-02-12T13:23:37.535910Z",
     "iopub.status.idle": "2024-02-12T13:24:16.327233Z",
     "shell.execute_reply": "2024-02-12T13:24:16.325584Z"
    },
    "papermill": {
     "duration": 38.799817,
     "end_time": "2024-02-12T13:24:16.330128",
     "exception": false,
     "start_time": "2024-02-12T13:23:37.530311",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install -q /kaggle/input/tabpfn-019-whl/tabpfn-0.1.9-py3-none-any.whl\n",
    "!mkdir /opt/conda/lib/python3.10/site-packages/tabpfn/models_diff\n",
    "!cp /kaggle/input/tabpfn-019-whl/prior_diff_real_checkpoint_n_0_epoch_100.cpkt /opt/conda/lib/python3.10/site-packages/tabpfn/models_diff/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1014883d",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2024-02-12T13:24:16.343585Z",
     "iopub.status.busy": "2024-02-12T13:24:16.342590Z",
     "iopub.status.idle": "2024-02-12T13:24:23.026177Z",
     "shell.execute_reply": "2024-02-12T13:24:23.024719Z"
    },
    "papermill": {
     "duration": 6.693885,
     "end_time": "2024-02-12T13:24:23.029697",
     "exception": false,
     "start_time": "2024-02-12T13:24:16.335812",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# importing necessary libraries\n",
    "import pandas as pd, numpy as np, catboost as ctb, pandas as pd, time, warnings\n",
    "\n",
    "from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV\n",
    "from sklearn.metrics import make_scorer, log_loss, balanced_accuracy_score, make_scorer, f1_score\n",
    "from sklearn import svm, neighbors, naive_bayes, ensemble, neural_network, preprocessing, impute, gaussian_process, linear_model, compose, cluster\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "from tabpfn import TabPFNClassifier\n",
    "from joblib import dump"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb4425d4",
   "metadata": {
    "papermill": {
     "duration": 0.004368,
     "end_time": "2024-02-12T13:24:23.038856",
     "exception": false,
     "start_time": "2024-02-12T13:24:23.034488",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Loading Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "15efdfbe",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-12T13:24:23.050433Z",
     "iopub.status.busy": "2024-02-12T13:24:23.049745Z",
     "iopub.status.idle": "2024-02-12T13:24:23.201905Z",
     "shell.execute_reply": "2024-02-12T13:24:23.200587Z"
    },
    "papermill": {
     "duration": 0.160974,
     "end_time": "2024-02-12T13:24:23.204456",
     "exception": false,
     "start_time": "2024-02-12T13:24:23.043482",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Alpha</th>\n",
       "      <th>Beta</th>\n",
       "      <th>Gamma</th>\n",
       "      <th>Delta</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>B</td>\n",
       "      <td>C</td>\n",
       "      <td>G</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A</td>\n",
       "      <td>C</td>\n",
       "      <td>M</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A</td>\n",
       "      <td>C</td>\n",
       "      <td>M</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A</td>\n",
       "      <td>C</td>\n",
       "      <td>M</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>D</td>\n",
       "      <td>B</td>\n",
       "      <td>F</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>612</th>\n",
       "      <td>A</td>\n",
       "      <td>B</td>\n",
       "      <td>M</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>613</th>\n",
       "      <td>A</td>\n",
       "      <td>B</td>\n",
       "      <td>M</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>614</th>\n",
       "      <td>A</td>\n",
       "      <td>C</td>\n",
       "      <td>M</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>615</th>\n",
       "      <td>A</td>\n",
       "      <td>C</td>\n",
       "      <td>M</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>616</th>\n",
       "      <td>A</td>\n",
       "      <td>C</td>\n",
       "      <td>M</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>617 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Alpha Beta Gamma Delta\n",
       "0       B    C     G     D\n",
       "1       A    C     M     B\n",
       "2       A    C     M     B\n",
       "3       A    C     M     B\n",
       "4       D    B     F     B\n",
       "..    ...  ...   ...   ...\n",
       "612     A    B     M     B\n",
       "613     A    B     M     B\n",
       "614     A    C     M     B\n",
       "615     A    C     M     B\n",
       "616     A    C     M     B\n",
       "\n",
       "[617 rows x 4 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Alpha_A</th>\n",
       "      <th>Alpha_B</th>\n",
       "      <th>Alpha_D</th>\n",
       "      <th>Alpha_G</th>\n",
       "      <th>Beta_A</th>\n",
       "      <th>Beta_B</th>\n",
       "      <th>Beta_C</th>\n",
       "      <th>Gamma_A</th>\n",
       "      <th>Gamma_B</th>\n",
       "      <th>Gamma_E</th>\n",
       "      <th>Gamma_F</th>\n",
       "      <th>Gamma_G</th>\n",
       "      <th>Gamma_H</th>\n",
       "      <th>Gamma_M</th>\n",
       "      <th>Gamma_N</th>\n",
       "      <th>Delta_A</th>\n",
       "      <th>Delta_B</th>\n",
       "      <th>Delta_C</th>\n",
       "      <th>Delta_D</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>612</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>613</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>614</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>615</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>616</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>617 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Alpha_A  Alpha_B  Alpha_D  Alpha_G  Beta_A  Beta_B  Beta_C  Gamma_A  \\\n",
       "0          0        1        0        0       0       0       1        0   \n",
       "1          1        0        0        0       0       0       1        0   \n",
       "2          1        0        0        0       0       0       1        0   \n",
       "3          1        0        0        0       0       0       1        0   \n",
       "4          0        0        1        0       0       1       0        0   \n",
       "..       ...      ...      ...      ...     ...     ...     ...      ...   \n",
       "612        1        0        0        0       0       1       0        0   \n",
       "613        1        0        0        0       0       1       0        0   \n",
       "614        1        0        0        0       0       0       1        0   \n",
       "615        1        0        0        0       0       0       1        0   \n",
       "616        1        0        0        0       0       0       1        0   \n",
       "\n",
       "     Gamma_B  Gamma_E  Gamma_F  Gamma_G  Gamma_H  Gamma_M  Gamma_N  Delta_A  \\\n",
       "0          0        0        0        1        0        0        0        0   \n",
       "1          0        0        0        0        0        1        0        0   \n",
       "2          0        0        0        0        0        1        0        0   \n",
       "3          0        0        0        0        0        1        0        0   \n",
       "4          0        0        1        0        0        0        0        0   \n",
       "..       ...      ...      ...      ...      ...      ...      ...      ...   \n",
       "612        0        0        0        0        0        1        0        0   \n",
       "613        0        0        0        0        0        1        0        0   \n",
       "614        0        0        0        0        0        1        0        0   \n",
       "615        0        0        0        0        0        1        0        0   \n",
       "616        0        0        0        0        0        1        0        0   \n",
       "\n",
       "     Delta_B  Delta_C  Delta_D  \n",
       "0          0        0        1  \n",
       "1          1        0        0  \n",
       "2          1        0        0  \n",
       "3          1        0        0  \n",
       "4          1        0        0  \n",
       "..       ...      ...      ...  \n",
       "612        1        0        0  \n",
       "613        1        0        0  \n",
       "614        1        0        0  \n",
       "615        1        0        0  \n",
       "616        1        0        0  \n",
       "\n",
       "[617 rows x 19 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(617, 19)\n"
     ]
    }
   ],
   "source": [
    "train = pd.read_csv(\"/kaggle/input/icr-identify-age-related-conditions/train.csv\")\n",
    "greeks = pd.read_csv(\"/kaggle/input/icr-identify-age-related-conditions/greeks.csv\")\n",
    "greeks = greeks.drop(['Epsilon'], axis = 1)\n",
    "train = pd.merge(train, greeks, on='Id', how='inner')\n",
    "classes = train[['Alpha', 'Beta', 'Gamma', 'Delta']]\n",
    "display(classes)\n",
    "classes = pd.get_dummies(classes, columns=['Alpha', 'Beta', 'Gamma', 'Delta'])\n",
    "class_columns = classes.columns\n",
    "display(classes)\n",
    "classes =  classes.to_numpy()\n",
    "print(np.shape(classes))\n",
    "train = train.drop(['Class', 'Id','Alpha', 'Beta', 'Gamma', 'Delta'], axis=1)\n",
    "features = train.columns\n",
    "\n",
    "numeric_transformer = Pipeline(\n",
    "    steps=[('imputer', impute.SimpleImputer(missing_values=pd.NA, strategy='mean')),\n",
    "           ('scaler', preprocessing.MinMaxScaler())])\n",
    "\n",
    "categoric_transformer = Pipeline(\n",
    "    steps=[('imputer', impute.SimpleImputer(missing_values=pd.NA, strategy='most_frequent')),\n",
    "           ('onehot', preprocessing.OneHotEncoder(handle_unknown='ignore'))])\n",
    "\n",
    "preprocessor = compose.ColumnTransformer(transformers=[\n",
    "        (\"num\", numeric_transformer, features.drop(['EJ'])),\n",
    "        (\"cat\", categoric_transformer, ['EJ'])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a4bc95fa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-12T13:24:23.217726Z",
     "iopub.status.busy": "2024-02-12T13:24:23.217285Z",
     "iopub.status.idle": "2024-02-12T13:24:23.224103Z",
     "shell.execute_reply": "2024-02-12T13:24:23.222549Z"
    },
    "papermill": {
     "duration": 0.017035,
     "end_time": "2024-02-12T13:24:23.227183",
     "exception": false,
     "start_time": "2024-02-12T13:24:23.210148",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "svc_weight = 1/10\n",
    "lr_weight = 1/26\n",
    "gp_weight = 1/28\n",
    "knn_weight = 1/25\n",
    "tab_weight = 1/30\n",
    "ctb_weight = 1/18"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "09136180",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-12T13:24:23.240472Z",
     "iopub.status.busy": "2024-02-12T13:24:23.240043Z",
     "iopub.status.idle": "2024-02-12T13:24:23.822606Z",
     "shell.execute_reply": "2024-02-12T13:24:23.821559Z"
    },
    "papermill": {
     "duration": 0.592311,
     "end_time": "2024-02-12T13:24:23.825276",
     "exception": false,
     "start_time": "2024-02-12T13:24:23.232965",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading model that can be used for inference only\n",
      "Using a Transformer with 25.82 M parameters\n"
     ]
    }
   ],
   "source": [
    "gp_pipe = Pipeline([(\"preprocessor\", preprocessor),\n",
    "                 ('gp', ensemble.BaggingClassifier(estimator=gaussian_process.GaussianProcessClassifier(max_iter_predict = 1000),n_estimators=100))]) #should be 800\n",
    "\n",
    "svc_pipe = Pipeline([(\"preprocessor\", preprocessor),\n",
    "                     ('svc', svm.SVC(max_iter = 10**9, C = 1, gamma = 1, kernel = 'poly', class_weight = 'balanced', probability = True))])\n",
    "\n",
    "lr_pipe = Pipeline([(\"preprocessor\", preprocessor),\n",
    "                     ('lr', ensemble.BaggingClassifier(estimator=linear_model.LogisticRegression(max_iter = 1000_000, solver = 'saga', class_weight = 'balanced'),n_estimators=1000))]) \n",
    "\n",
    "kn_pipe = Pipeline([(\"preprocessor\", preprocessor),\n",
    "                 ('kn', ensemble.BaggingClassifier(estimator=neighbors.KNeighborsClassifier(weights = 'distance', algorithm = 'kd_tree', metric = 'manhattan', n_neighbors = 5),n_estimators=1000, max_samples = 0.7))])\n",
    "ctb_pipe = Pipeline([(\"preprocessor\", preprocessor),\n",
    "                 ('ctb', ctb.CatBoostClassifier(n_estimators = 1000, learning_rate = 0.1, verbose = False))])\n",
    "\n",
    "tab_pipe = Pipeline([(\"preprocessor\", preprocessor),\n",
    "                 ('tab', TabPFNClassifier())]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "04bce5cf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-12T13:24:23.841587Z",
     "iopub.status.busy": "2024-02-12T13:24:23.841182Z",
     "iopub.status.idle": "2024-02-12T13:24:23.846256Z",
     "shell.execute_reply": "2024-02-12T13:24:23.844892Z"
    },
    "papermill": {
     "duration": 0.014809,
     "end_time": "2024-02-12T13:24:23.848666",
     "exception": false,
     "start_time": "2024-02-12T13:24:23.833857",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#lr_pipe = Pipeline([(\"preprocessor\", preprocessor),\n",
    "#                     ('lr', MultiOutputClassifier(ensemble.BaggingClassifier(estimator=linear_model.LogisticRegression(max_iter = 1000_000, solver = 'saga', class_weight = 'balanced'),n_estimators=1000), n_jobs = -1))]) \n",
    "#lr_pipe.fit(train, classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "74273dc4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-12T13:24:23.862374Z",
     "iopub.status.busy": "2024-02-12T13:24:23.861950Z",
     "iopub.status.idle": "2024-02-12T13:25:27.758152Z",
     "shell.execute_reply": "2024-02-12T13:25:27.756872Z"
    },
    "papermill": {
     "duration": 63.912338,
     "end_time": "2024-02-12T13:25:27.766867",
     "exception": false,
     "start_time": "2024-02-12T13:24:23.854529",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading model that can be used for inference only\n",
      "Using a Transformer with 25.82 M parameters\n",
      "Loading model that can be used for inference only\n",
      "Using a Transformer with 25.82 M parameters\n",
      "Loading model that can be used for inference only\n",
      "Using a Transformer with 25.82 M parameters\n",
      "Loading model that can be used for inference only\n",
      "Using a Transformer with 25.82 M parameters\n",
      "Loading model that can be used for inference only\n",
      "Using a Transformer with 25.82 M parameters\n",
      "Loading model that can be used for inference only\n",
      "Using a Transformer with 25.82 M parameters\n",
      "Loading model that can be used for inference only\n",
      "Using a Transformer with 25.82 M parameters\n",
      "Loading model that can be used for inference only\n",
      "Using a Transformer with 25.82 M parameters\n",
      "Loading model that can be used for inference only\n",
      "Using a Transformer with 25.82 M parameters\n",
      "Loading model that can be used for inference only\n",
      "Using a Transformer with 25.82 M parameters\n",
      "Loading model that can be used for inference only\n",
      "Using a Transformer with 25.82 M parameters\n",
      "Loading model that can be used for inference only\n",
      "Using a Transformer with 25.82 M parameters\n",
      "Loading model that can be used for inference only\n",
      "Using a Transformer with 25.82 M parameters\n",
      "Loading model that can be used for inference only\n",
      "Using a Transformer with 25.82 M parameters\n",
      "Loading model that can be used for inference only\n",
      "Using a Transformer with 25.82 M parameters\n",
      "Loading model that can be used for inference only\n",
      "Using a Transformer with 25.82 M parameters\n",
      "Loading model that can be used for inference only\n",
      "Using a Transformer with 25.82 M parameters\n",
      "Loading model that can be used for inference only\n",
      "Using a Transformer with 25.82 M parameters\n",
      "Loading model that can be used for inference only\n",
      "Using a Transformer with 25.82 M parameters\n",
      "Loading model that can be used for inference only\n",
      "Using a Transformer with 25.82 M parameters\n",
      "2\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Alpha</th>\n",
       "      <th>Beta</th>\n",
       "      <th>Gamma</th>\n",
       "      <th>Delta</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>B</td>\n",
       "      <td>C</td>\n",
       "      <td>G</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A</td>\n",
       "      <td>C</td>\n",
       "      <td>M</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A</td>\n",
       "      <td>C</td>\n",
       "      <td>M</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A</td>\n",
       "      <td>C</td>\n",
       "      <td>M</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>D</td>\n",
       "      <td>B</td>\n",
       "      <td>F</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>612</th>\n",
       "      <td>A</td>\n",
       "      <td>B</td>\n",
       "      <td>M</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>613</th>\n",
       "      <td>A</td>\n",
       "      <td>B</td>\n",
       "      <td>M</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>614</th>\n",
       "      <td>A</td>\n",
       "      <td>C</td>\n",
       "      <td>M</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>615</th>\n",
       "      <td>A</td>\n",
       "      <td>C</td>\n",
       "      <td>M</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>616</th>\n",
       "      <td>A</td>\n",
       "      <td>C</td>\n",
       "      <td>M</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>617 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Alpha Beta Gamma Delta\n",
       "0       B    C     G     D\n",
       "1       A    C     M     B\n",
       "2       A    C     M     B\n",
       "3       A    C     M     B\n",
       "4       D    B     F     B\n",
       "..    ...  ...   ...   ...\n",
       "612     A    B     M     B\n",
       "613     A    B     M     B\n",
       "614     A    C     M     B\n",
       "615     A    C     M     B\n",
       "616     A    C     M     B\n",
       "\n",
       "[617 rows x 4 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#with warnings.catch_warnings():\n",
    "#    warnings.simplefilter(\"ignore\")\n",
    "#    ensemble_model = MultiOutputClassifier(ensemble.VotingClassifier(estimators=[('svc', svc_pipe), ('lr', lr_pipe), ('gp', gp_pipe), ('knn', kn_pipe),('tab_pipe', tab_pipe), ('ctb_pipe', ctb_pipe)],voting='soft', weights =[svc_weight, lr_weight, gp_weight, knn_weight, tab_weight, ctb_weight]), n_jobs = -1) #0.207\n",
    "#    #ensemble_model = MultiOutputClassifier(ensemble.VotingClassifier(estimators=[('svc', svc_pipe),('tab_pipe', tab_pipe), ('ctb_pipe', ctb_pipe)],voting='soft', weights =[svc_weight, tab_weight, ctb_weight]), n_jobs = -1) #0.207\n",
    "#    ensemble_model.fit(train, classes)\n",
    "svc_model = Pipeline([(\"preprocessor\", preprocessor),\n",
    "                     ('svc', MultiOutputClassifier(svm.SVC(max_iter = 10**9, C = 1, gamma = 1, kernel = 'poly', class_weight = 'balanced', probability = True)))])\n",
    "tab_model = Pipeline([(\"preprocessor\", preprocessor),\n",
    "                 ('tab', MultiOutputClassifier(TabPFNClassifier()))]);\n",
    "ensemble_model = tab_model\n",
    "ensemble_model.fit(train, classes)\n",
    "predictions = np.array(ensemble_model.predict_proba(train))\n",
    "predictions = pd.DataFrame(predictions[:, :, 1].T, columns= class_columns)\n",
    "alphas = ['Alpha_A','Alpha_B','Alpha_D','Alpha_G']\n",
    "betas = ['Beta_A','Beta_B','Beta_C']\n",
    "gammas = ['Gamma_A','Gamma_B','Gamma_E','Gamma_F','Gamma_G','Gamma_H','Gamma_M','Gamma_N']\n",
    "deltas = ['Delta_A','Delta_B','Delta_C','Delta_D']\n",
    "\n",
    "def singularize(dataframe, columns):\n",
    "    prediction = predictions[columns]\n",
    "    arr = prediction.to_numpy()\n",
    "    max_indices = np.argmax(arr, axis=1)\n",
    "    result = np.zeros_like(arr)\n",
    "    result[np.arange(result.shape[0]), max_indices] = 1\n",
    "    return pd.DataFrame(result, columns = columns).astype('int32')\n",
    "\n",
    "alphas = singularize(predictions, alphas)\n",
    "betas = singularize(predictions, betas)\n",
    "gammas = singularize(predictions, gammas)\n",
    "deltas = singularize(predictions, deltas)\n",
    "\n",
    "new_classes = alphas.join([betas, gammas, deltas])\n",
    "print(np.sum(np.square(new_classes.to_numpy() - classes)))\n",
    "\n",
    "categories = new_classes.columns.str.split('_').str[0].unique()\n",
    "collapsed_df = pd.DataFrame({category: new_classes[[col for col in new_classes.columns if category in col]].idxmax(axis=1).str.split('_').str[1] for category in categories})\n",
    "display(collapsed_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "94329c36",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-12T13:25:27.785519Z",
     "iopub.status.busy": "2024-02-12T13:25:27.784471Z",
     "iopub.status.idle": "2024-02-12T13:25:34.112817Z",
     "shell.execute_reply": "2024-02-12T13:25:34.111941Z"
    },
    "papermill": {
     "duration": 6.340092,
     "end_time": "2024-02-12T13:25:34.115136",
     "exception": false,
     "start_time": "2024-02-12T13:25:27.775044",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['greeks_augment_model.joblib']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dump(ensemble_model, 'greeks_augment_model.joblib')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aae704cc",
   "metadata": {
    "papermill": {
     "duration": 0.008036,
     "end_time": "2024-02-12T13:25:34.131723",
     "exception": false,
     "start_time": "2024-02-12T13:25:34.123687",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "none",
   "dataSources": [
    {
     "databundleVersionId": 5687476,
     "sourceId": 52784,
     "sourceType": "competition"
    },
    {
     "datasetId": 3327426,
     "sourceId": 5794968,
     "sourceType": "datasetVersion"
    }
   ],
   "dockerImageVersionId": 30513,
   "isGpuEnabled": false,
   "isInternetEnabled": false,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 132.107788,
   "end_time": "2024-02-12T13:25:35.966653",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-02-12T13:23:23.858865",
   "version": "2.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}