TODO_physionet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "from mne import Epochs, pick_types, events_from_annotations\n",
    "from mne.io.edf import read_raw_edf\n",
    "from mne.datasets import eegbci\n",
    "from mne.decoding import CSP\n",
    "from sklearn.model_selection import cross_val_score, KFold\n",
    "\n",
    "import torch\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import f1_score, cohen_kappa_score\n",
    "from sklearn.model_selection import KFold\n",
    "from kan.KAN import KAN \n",
    "import mne\n",
    "from mne.datasets import eegbci  \n",
    "from mne import EpochsArray, pick_types, events_from_annotations\n",
    "from mne.decoding import CSP  \n",
    "from torch.nn import Sequential, Linear, ReLU, CrossEntropyLoss\n",
    "from torch.optim import LBFGS \n",
    "from kan.KAN import KAN \n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "torch.set_default_dtype(torch.float64)\n",
    "from mrmr import mrmr_classif\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# avoid classification of evoked responses by using epochs that start 1s after\n",
    "# cue onset.\n",
    "def calculate_metrics(output, target):\n",
    "    preds = output.argmax(dim=1)\n",
    "    accuracy = (preds == target).sum().item() / len(target)\n",
    "    f1 = f1_score(target.cpu(), preds.cpu(), average='weighted') \n",
    "    kappa = cohen_kappa_score(target.cpu(), preds.cpu())\n",
    "    return accuracy, f1, kappa\n",
    "\n",
    "def my_loss_fn(output, target):\n",
    "    loss = CrossEntropyLoss()(output, target)\n",
    "    return loss.double()\n",
    "\n",
    "def iteration_subjects(subjects, runs,event_id):\n",
    "    tmin, tmax = -1., 3.\n",
    "    epochs_data=[]\n",
    "    for subject in subjects:\n",
    "        for data in eegbci.load_data(subject, runs):\n",
    "            raw=read_raw_edf(data, preload=True)\n",
    "            picks = pick_types(\n",
    "                            raw.info, meg=False, eeg=True, stim=False, eog=False, exclude='bads')\n",
    "\n",
    "            raw.filter(7., 35., method='iir', picks=picks)\n",
    "            events, _ = events_from_annotations(raw, event_id=event_id)\n",
    "            epochs = Epochs(\n",
    "                        raw,\n",
    "                        events,\n",
    "                        event_id,\n",
    "                        tmin,\n",
    "                        tmax,\n",
    "                        proj=True,\n",
    "                        picks=picks,\n",
    "                        baseline=None,\n",
    "                        preload=True,\n",
    "                        verbose=False)\n",
    "            if event_id['T1']>2:\n",
    "                epochs.event_id = {'T3': 3, 'T4': 4}\n",
    "            epochs_data.append(epochs)\n",
    "    return epochs_data\n",
    "\n",
    "\n",
    "def data_EEG(train=True, multiclass=False):\n",
    "    #6, 10, 14 \tMotor imagery: hands vs feet\n",
    "    #4, 8, 12   Motor imagery: left vs right hand\n",
    "\n",
    "    event_id = dict(T1=1, T2=2)\n",
    "    subjects = [1]  # [77, 16, 62, 9, 46, 30, 24, 84, 56, 95]\n",
    "    if train:\n",
    "        runs = [4, 8]  # motor imagery: left - right \n",
    "    else:\n",
    "        runs = [12]\n",
    "    epochs_data = iteration_subjects(subjects, runs,event_id)\n",
    "    if multiclass:\n",
    "        event_id = dict(T1=3, T2=4)\n",
    "        if train:\n",
    "            runs = [6, 10]  # motor imagery: left - right \n",
    "        else:\n",
    "            runs = [14]\n",
    "        epochs_data.extend(iteration_subjects(subjects, runs,event_id))\n",
    "        return mne.concatenate_epochs(epochs_data)[['T1', 'T2', 'T4']]\n",
    "\n",
    "    \n",
    "    return mne.concatenate_epochs(epochs_data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def creating_models(input_shape, output_shape):\n",
    "    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "    kan_model = KAN(width=[input_shape, 5, 5, output_shape], grid=100, k=10, seed=0, device=device)             \n",
    "    for param in kan_model.parameters():\n",
    "        param.data = param.data.double()\n",
    "    mlp_model = Sequential(\n",
    "      Linear(input_shape, 8).double(),  \n",
    "      ReLU(),\n",
    "      Linear(8, output_shape).double()  # 2 output classes \n",
    "      )\n",
    "\n",
    "    for param in mlp_model.parameters():\n",
    "        param.data = param.data.double()\n",
    "        \n",
    "    return kan_model,mlp_model\n",
    "\n",
    "\n",
    "def train_model(model, optimizer, X_train, y_train):\n",
    "    optimizer.zero_grad()\n",
    "    output = model(X_train)\n",
    "    loss = CrossEntropyLoss()(output, y_train) \n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "import torch \n",
    "from torch.utils.data import Dataset\n",
    "\n",
    "class EEGDataset(Dataset):\n",
    "    def __init__(self, X, y):\n",
    "        self.X = torch.from_numpy(X).double() # Or .float() if single precision is sufficient\n",
    "        self.y = torch.from_numpy(y).long()\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.X)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        return self.X[index], self.y[index]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Raw - Holdout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading EEGBCI data\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading file 'S001/S001R04.edf' from 'https://physionet.org/files/eegmmidb/1.0.0/S001/S001R04.edf' to 'C:\\Users\\JARS\\datasets\\MNE-eegbci-data\\files\\eegmmidb\\1.0.0'.\n",
      "Downloading file 'S001/S001R08.edf' from 'https://physionet.org/files/eegmmidb/1.0.0/S001/S001R08.edf' to 'C:\\Users\\JARS\\datasets\\MNE-eegbci-data\\files\\eegmmidb\\1.0.0'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Download complete in 23s (5.0 MB)\n",
      "Extracting EDF parameters from C:\\Users\\JARS\\datasets\\MNE-eegbci-data\\files\\eegmmidb\\1.0.0\\S001\\S001R04.edf...\n",
      "EDF file detected\n",
      "Setting channel info structure...\n",
      "Creating raw.info structure...\n",
      "Reading 0 ... 19999  =      0.000 ...   124.994 secs...\n",
      "Filtering raw data in 1 contiguous segment\n",
      "Setting up band-pass filter from 7 - 35 Hz\n",
      "\n",
      "IIR filter parameters\n",
      "---------------------\n",
      "Butterworth bandpass zero-phase (two-pass forward and reverse) non-causal filter:\n",
      "- Filter order 16 (effective, after forward-backward)\n",
      "- Cutoffs at 7.00, 35.00 Hz: -6.02, -6.02 dB\n",
      "\n",
      "Used Annotations descriptions: ['T1', 'T2']\n",
      "Extracting EDF parameters from C:\\Users\\JARS\\datasets\\MNE-eegbci-data\\files\\eegmmidb\\1.0.0\\S001\\S001R08.edf...\n",
      "EDF file detected\n",
      "Setting channel info structure...\n",
      "Creating raw.info structure...\n",
      "Reading 0 ... 19999  =      0.000 ...   124.994 secs...\n",
      "Filtering raw data in 1 contiguous segment\n",
      "Setting up band-pass filter from 7 - 35 Hz\n",
      "\n",
      "IIR filter parameters\n",
      "---------------------\n",
      "Butterworth bandpass zero-phase (two-pass forward and reverse) non-causal filter:\n",
      "- Filter order 16 (effective, after forward-backward)\n",
      "- Cutoffs at 7.00, 35.00 Hz: -6.02, -6.02 dB\n",
      "\n",
      "Used Annotations descriptions: ['T1', 'T2']\n",
      "Not setting metadata\n",
      "30 matching events found\n",
      "No baseline correction applied\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10/10 [00:00<00:00, 33.60it/s]\n"
     ]
    }
   ],
   "source": [
    "df= data_EEG().to_data_frame().drop(columns=['time', 'epoch'])\n",
    "# Data Conversion (NumPy to PyTorch)\n",
    "selected_features = mrmr_classif(X=df.drop(columns=['condition']), y=df.condition, K=10)\n",
    "df_features = df[selected_features+['condition']]\n",
    "X = df.drop(columns=['condition']).values\n",
    "y = df.condition.values\n",
    "le = LabelEncoder()\n",
    "y = le.fit_transform(y)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)\n",
    "\n",
    "X_train = torch.from_numpy(X_train).double()\n",
    "X_test  = torch.from_numpy(X_test).double()\n",
    "y_train  = torch.from_numpy(y_train).long()\n",
    "y_test  = torch.from_numpy(y_test).long()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50 | KAN Train Acc: 0.535 | KAN F1: 0.376 | KAN Kappa: 0.001 | KAN Test Acc: 0.529 | KAN Test F1: 0.368 | KAN Test Kappa: -0.000 | \n",
      "Epoch 1/50 | MLP Train Acc: 0.520 | MLP F1: 0.520 | MLP Kappa: 0.037 | MLP Test Acc: 0.517 | MLP Test F1: 0.517 | MLP Test Kappa: 0.030 | \n",
      "Epoch 2/50 | KAN Train Acc: 0.537 | KAN F1: 0.420 | KAN Kappa: 0.012 | KAN Test Acc: 0.525 | KAN Test F1: 0.409 | KAN Test Kappa: -0.000 | \n",
      "Epoch 2/50 | MLP Train Acc: 0.544 | MLP F1: 0.485 | MLP Kappa: 0.042 | MLP Test Acc: 0.536 | MLP Test F1: 0.478 | MLP Test Kappa: 0.035 | \n",
      "Epoch 3/50 | KAN Train Acc: 0.538 | KAN F1: 0.391 | KAN Kappa: 0.008 | KAN Test Acc: 0.527 | KAN Test F1: 0.376 | KAN Test Kappa: -0.002 | \n",
      "Epoch 3/50 | MLP Train Acc: 0.554 | MLP F1: 0.551 | MLP Kappa: 0.096 | MLP Test Acc: 0.540 | MLP Test F1: 0.537 | MLP Test Kappa: 0.071 | \n",
      "Epoch 4/50 | KAN Train Acc: 0.546 | KAN F1: 0.433 | KAN Kappa: 0.032 | KAN Test Acc: 0.525 | KAN Test F1: 0.409 | KAN Test Kappa: -0.002 | \n",
      "Epoch 4/50 | MLP Train Acc: 0.568 | MLP F1: 0.566 | MLP Kappa: 0.126 | MLP Test Acc: 0.558 | MLP Test F1: 0.555 | MLP Test Kappa: 0.108 | \n",
      "Epoch 5/50 | KAN Train Acc: 0.554 | KAN F1: 0.477 | KAN Kappa: 0.058 | KAN Test Acc: 0.523 | KAN Test F1: 0.443 | KAN Test Kappa: 0.003 | \n",
      "Epoch 5/50 | MLP Train Acc: 0.577 | MLP F1: 0.576 | MLP Kappa: 0.146 | MLP Test Acc: 0.553 | MLP Test F1: 0.551 | MLP Test Kappa: 0.098 | \n",
      "Epoch 6/50 | KAN Train Acc: 0.561 | KAN F1: 0.488 | KAN Kappa: 0.073 | KAN Test Acc: 0.523 | KAN Test F1: 0.445 | KAN Test Kappa: 0.003 | \n",
      "Epoch 6/50 | MLP Train Acc: 0.585 | MLP F1: 0.585 | MLP Kappa: 0.165 | MLP Test Acc: 0.565 | MLP Test F1: 0.564 | MLP Test Kappa: 0.125 | \n",
      "Epoch 7/50 | KAN Train Acc: 0.562 | KAN F1: 0.499 | KAN Kappa: 0.078 | KAN Test Acc: 0.524 | KAN Test F1: 0.458 | KAN Test Kappa: 0.009 | \n",
      "Epoch 7/50 | MLP Train Acc: 0.590 | MLP F1: 0.581 | MLP Kappa: 0.162 | MLP Test Acc: 0.575 | MLP Test F1: 0.567 | MLP Test Kappa: 0.136 | \n",
      "Epoch 8/50 | KAN Train Acc: 0.569 | KAN F1: 0.509 | KAN Kappa: 0.093 | KAN Test Acc: 0.521 | KAN Test F1: 0.457 | KAN Test Kappa: 0.004 | \n",
      "Epoch 8/50 | MLP Train Acc: 0.599 | MLP F1: 0.589 | MLP Kappa: 0.179 | MLP Test Acc: 0.581 | MLP Test F1: 0.572 | MLP Test Kappa: 0.147 | \n",
      "Epoch 9/50 | KAN Train Acc: 0.571 | KAN F1: 0.520 | KAN Kappa: 0.100 | KAN Test Acc: 0.526 | KAN Test F1: 0.469 | KAN Test Kappa: 0.016 | \n",
      "Epoch 9/50 | MLP Train Acc: 0.607 | MLP F1: 0.601 | MLP Kappa: 0.200 | MLP Test Acc: 0.579 | MLP Test F1: 0.573 | MLP Test Kappa: 0.146 | \n",
      "Epoch 10/50 | KAN Train Acc: 0.572 | KAN F1: 0.533 | KAN Kappa: 0.108 | KAN Test Acc: 0.528 | KAN Test F1: 0.485 | KAN Test Kappa: 0.024 | \n",
      "Epoch 10/50 | MLP Train Acc: 0.612 | MLP F1: 0.606 | MLP Kappa: 0.210 | MLP Test Acc: 0.582 | MLP Test F1: 0.576 | MLP Test Kappa: 0.152 | \n",
      "Epoch 11/50 | KAN Train Acc: 0.583 | KAN F1: 0.536 | KAN Kappa: 0.126 | KAN Test Acc: 0.524 | KAN Test F1: 0.473 | KAN Test Kappa: 0.013 | \n",
      "Epoch 11/50 | MLP Train Acc: 0.615 | MLP F1: 0.609 | MLP Kappa: 0.215 | MLP Test Acc: 0.582 | MLP Test F1: 0.575 | MLP Test Kappa: 0.152 | \n",
      "Epoch 12/50 | KAN Train Acc: 0.584 | KAN F1: 0.555 | KAN Kappa: 0.136 | KAN Test Acc: 0.521 | KAN Test F1: 0.489 | KAN Test Kappa: 0.014 | \n",
      "Epoch 12/50 | MLP Train Acc: 0.616 | MLP F1: 0.611 | MLP Kappa: 0.220 | MLP Test Acc: 0.580 | MLP Test F1: 0.574 | MLP Test Kappa: 0.148 | \n",
      "Epoch 13/50 | KAN Train Acc: 0.588 | KAN F1: 0.562 | KAN Kappa: 0.147 | KAN Test Acc: 0.523 | KAN Test F1: 0.494 | KAN Test Kappa: 0.019 | \n",
      "Epoch 13/50 | MLP Train Acc: 0.620 | MLP F1: 0.615 | MLP Kappa: 0.227 | MLP Test Acc: 0.582 | MLP Test F1: 0.576 | MLP Test Kappa: 0.152 | \n",
      "Epoch 14/50 | KAN Train Acc: 0.589 | KAN F1: 0.568 | KAN Kappa: 0.152 | KAN Test Acc: 0.517 | KAN Test F1: 0.492 | KAN Test Kappa: 0.008 | \n",
      "Epoch 14/50 | MLP Train Acc: 0.623 | MLP F1: 0.619 | MLP Kappa: 0.234 | MLP Test Acc: 0.585 | MLP Test F1: 0.579 | MLP Test Kappa: 0.158 | \n",
      "Epoch 15/50 | KAN Train Acc: 0.592 | KAN F1: 0.575 | KAN Kappa: 0.159 | KAN Test Acc: 0.517 | KAN Test F1: 0.497 | KAN Test Kappa: 0.012 | \n",
      "Epoch 15/50 | MLP Train Acc: 0.627 | MLP F1: 0.622 | MLP Kappa: 0.241 | MLP Test Acc: 0.584 | MLP Test F1: 0.578 | MLP Test Kappa: 0.156 | \n",
      "Epoch 16/50 | KAN Train Acc: 0.595 | KAN F1: 0.578 | KAN Kappa: 0.166 | KAN Test Acc: 0.518 | KAN Test F1: 0.499 | KAN Test Kappa: 0.014 | \n",
      "Epoch 16/50 | MLP Train Acc: 0.630 | MLP F1: 0.630 | MLP Kappa: 0.255 | MLP Test Acc: 0.584 | MLP Test F1: 0.583 | MLP Test Kappa: 0.163 | \n",
      "Epoch 17/50 | KAN Train Acc: 0.600 | KAN F1: 0.588 | KAN Kappa: 0.179 | KAN Test Acc: 0.518 | KAN Test F1: 0.506 | KAN Test Kappa: 0.018 | \n",
      "Epoch 17/50 | MLP Train Acc: 0.633 | MLP F1: 0.633 | MLP Kappa: 0.261 | MLP Test Acc: 0.587 | MLP Test F1: 0.586 | MLP Test Kappa: 0.169 | \n",
      "Epoch 18/50 | KAN Train Acc: 0.604 | KAN F1: 0.596 | KAN Kappa: 0.192 | KAN Test Acc: 0.521 | KAN Test F1: 0.512 | KAN Test Kappa: 0.026 | \n",
      "Epoch 18/50 | MLP Train Acc: 0.636 | MLP F1: 0.635 | MLP Kappa: 0.266 | MLP Test Acc: 0.585 | MLP Test F1: 0.584 | MLP Test Kappa: 0.164 | \n",
      "Epoch 19/50 | KAN Train Acc: 0.608 | KAN F1: 0.601 | KAN Kappa: 0.201 | KAN Test Acc: 0.520 | KAN Test F1: 0.511 | KAN Test Kappa: 0.024 | \n",
      "Epoch 19/50 | MLP Train Acc: 0.637 | MLP F1: 0.637 | MLP Kappa: 0.269 | MLP Test Acc: 0.594 | MLP Test F1: 0.593 | MLP Test Kappa: 0.183 | \n",
      "Epoch 20/50 | KAN Train Acc: 0.611 | KAN F1: 0.603 | KAN Kappa: 0.206 | KAN Test Acc: 0.520 | KAN Test F1: 0.512 | KAN Test Kappa: 0.025 | \n",
      "Epoch 20/50 | MLP Train Acc: 0.639 | MLP F1: 0.639 | MLP Kappa: 0.274 | MLP Test Acc: 0.589 | MLP Test F1: 0.588 | MLP Test Kappa: 0.173 | \n",
      "Epoch 21/50 | KAN Train Acc: 0.611 | KAN F1: 0.601 | KAN Kappa: 0.204 | KAN Test Acc: 0.519 | KAN Test F1: 0.506 | KAN Test Kappa: 0.019 | \n",
      "Epoch 21/50 | MLP Train Acc: 0.642 | MLP F1: 0.642 | MLP Kappa: 0.280 | MLP Test Acc: 0.590 | MLP Test F1: 0.589 | MLP Test Kappa: 0.176 | \n",
      "Epoch 22/50 | KAN Train Acc: 0.615 | KAN F1: 0.601 | KAN Kappa: 0.208 | KAN Test Acc: 0.523 | KAN Test F1: 0.506 | KAN Test Kappa: 0.024 | \n",
      "Epoch 22/50 | MLP Train Acc: 0.640 | MLP F1: 0.640 | MLP Kappa: 0.275 | MLP Test Acc: 0.589 | MLP Test F1: 0.588 | MLP Test Kappa: 0.173 | \n",
      "Epoch 23/50 | KAN Train Acc: 0.617 | KAN F1: 0.603 | KAN Kappa: 0.213 | KAN Test Acc: 0.526 | KAN Test F1: 0.510 | KAN Test Kappa: 0.031 | \n",
      "Epoch 23/50 | MLP Train Acc: 0.642 | MLP F1: 0.641 | MLP Kappa: 0.279 | MLP Test Acc: 0.587 | MLP Test F1: 0.586 | MLP Test Kappa: 0.169 | \n",
      "Epoch 24/50 | KAN Train Acc: 0.619 | KAN F1: 0.605 | KAN Kappa: 0.217 | KAN Test Acc: 0.521 | KAN Test F1: 0.505 | KAN Test Kappa: 0.022 | \n",
      "Epoch 24/50 | MLP Train Acc: 0.643 | MLP F1: 0.642 | MLP Kappa: 0.281 | MLP Test Acc: 0.585 | MLP Test F1: 0.585 | MLP Test Kappa: 0.166 | \n",
      "Epoch 25/50 | KAN Train Acc: 0.621 | KAN F1: 0.607 | KAN Kappa: 0.221 | KAN Test Acc: 0.521 | KAN Test F1: 0.505 | KAN Test Kappa: 0.022 | \n",
      "Epoch 25/50 | MLP Train Acc: 0.645 | MLP F1: 0.645 | MLP Kappa: 0.285 | MLP Test Acc: 0.586 | MLP Test F1: 0.586 | MLP Test Kappa: 0.168 | \n",
      "Epoch 26/50 | KAN Train Acc: 0.624 | KAN F1: 0.614 | KAN Kappa: 0.230 | KAN Test Acc: 0.517 | KAN Test F1: 0.504 | KAN Test Kappa: 0.015 | \n",
      "Epoch 26/50 | MLP Train Acc: 0.643 | MLP F1: 0.643 | MLP Kappa: 0.282 | MLP Test Acc: 0.587 | MLP Test F1: 0.586 | MLP Test Kappa: 0.170 | \n",
      "Epoch 27/50 | KAN Train Acc: 0.627 | KAN F1: 0.616 | KAN Kappa: 0.235 | KAN Test Acc: 0.522 | KAN Test F1: 0.508 | KAN Test Kappa: 0.024 | \n",
      "Epoch 27/50 | MLP Train Acc: 0.645 | MLP F1: 0.645 | MLP Kappa: 0.286 | MLP Test Acc: 0.586 | MLP Test F1: 0.585 | MLP Test Kappa: 0.167 | \n",
      "Epoch 28/50 | KAN Train Acc: 0.630 | KAN F1: 0.614 | KAN Kappa: 0.237 | KAN Test Acc: 0.524 | KAN Test F1: 0.506 | KAN Test Kappa: 0.026 | \n",
      "Epoch 28/50 | MLP Train Acc: 0.642 | MLP F1: 0.642 | MLP Kappa: 0.280 | MLP Test Acc: 0.592 | MLP Test F1: 0.592 | MLP Test Kappa: 0.181 | \n",
      "Epoch 29/50 | KAN Train Acc: 0.631 | KAN F1: 0.615 | KAN Kappa: 0.240 | KAN Test Acc: 0.526 | KAN Test F1: 0.506 | KAN Test Kappa: 0.029 | \n",
      "Epoch 29/50 | MLP Train Acc: 0.647 | MLP F1: 0.647 | MLP Kappa: 0.289 | MLP Test Acc: 0.595 | MLP Test F1: 0.595 | MLP Test Kappa: 0.187 | \n",
      "Epoch 30/50 | KAN Train Acc: 0.635 | KAN F1: 0.610 | KAN Kappa: 0.243 | KAN Test Acc: 0.522 | KAN Test F1: 0.492 | KAN Test Kappa: 0.016 | \n",
      "Epoch 30/50 | MLP Train Acc: 0.648 | MLP F1: 0.648 | MLP Kappa: 0.292 | MLP Test Acc: 0.593 | MLP Test F1: 0.593 | MLP Test Kappa: 0.183 | \n",
      "Epoch 31/50 | KAN Train Acc: 0.639 | KAN F1: 0.613 | KAN Kappa: 0.251 | KAN Test Acc: 0.523 | KAN Test F1: 0.490 | KAN Test Kappa: 0.017 | \n",
      "Epoch 31/50 | MLP Train Acc: 0.648 | MLP F1: 0.647 | MLP Kappa: 0.291 | MLP Test Acc: 0.594 | MLP Test F1: 0.593 | MLP Test Kappa: 0.183 | \n",
      "Epoch 32/50 | KAN Train Acc: 0.638 | KAN F1: 0.614 | KAN Kappa: 0.250 | KAN Test Acc: 0.524 | KAN Test F1: 0.494 | KAN Test Kappa: 0.021 | \n",
      "Epoch 32/50 | MLP Train Acc: 0.648 | MLP F1: 0.647 | MLP Kappa: 0.291 | MLP Test Acc: 0.597 | MLP Test F1: 0.597 | MLP Test Kappa: 0.190 | \n",
      "Epoch 33/50 | KAN Train Acc: 0.640 | KAN F1: 0.614 | KAN Kappa: 0.253 | KAN Test Acc: 0.520 | KAN Test F1: 0.486 | KAN Test Kappa: 0.011 | \n",
      "Epoch 33/50 | MLP Train Acc: 0.648 | MLP F1: 0.648 | MLP Kappa: 0.292 | MLP Test Acc: 0.598 | MLP Test F1: 0.598 | MLP Test Kappa: 0.192 | \n",
      "Epoch 34/50 | KAN Train Acc: 0.642 | KAN F1: 0.616 | KAN Kappa: 0.256 | KAN Test Acc: 0.521 | KAN Test F1: 0.487 | KAN Test Kappa: 0.013 | \n",
      "Epoch 34/50 | MLP Train Acc: 0.649 | MLP F1: 0.649 | MLP Kappa: 0.294 | MLP Test Acc: 0.599 | MLP Test F1: 0.598 | MLP Test Kappa: 0.193 | \n",
      "Epoch 35/50 | KAN Train Acc: 0.646 | KAN F1: 0.622 | KAN Kappa: 0.266 | KAN Test Acc: 0.521 | KAN Test F1: 0.490 | KAN Test Kappa: 0.014 | \n",
      "Epoch 35/50 | MLP Train Acc: 0.649 | MLP F1: 0.649 | MLP Kappa: 0.293 | MLP Test Acc: 0.598 | MLP Test F1: 0.597 | MLP Test Kappa: 0.192 | \n",
      "Epoch 36/50 | KAN Train Acc: 0.645 | KAN F1: 0.628 | KAN Kappa: 0.268 | KAN Test Acc: 0.521 | KAN Test F1: 0.500 | KAN Test Kappa: 0.019 | \n",
      "Epoch 36/50 | MLP Train Acc: 0.649 | MLP F1: 0.649 | MLP Kappa: 0.294 | MLP Test Acc: 0.595 | MLP Test F1: 0.595 | MLP Test Kappa: 0.187 | \n",
      "Epoch 37/50 | KAN Train Acc: 0.647 | KAN F1: 0.635 | KAN Kappa: 0.276 | KAN Test Acc: 0.522 | KAN Test F1: 0.506 | KAN Test Kappa: 0.024 | \n",
      "Epoch 37/50 | MLP Train Acc: 0.650 | MLP F1: 0.650 | MLP Kappa: 0.296 | MLP Test Acc: 0.598 | MLP Test F1: 0.597 | MLP Test Kappa: 0.192 | \n",
      "Epoch 38/50 | KAN Train Acc: 0.647 | KAN F1: 0.629 | KAN Kappa: 0.272 | KAN Test Acc: 0.520 | KAN Test F1: 0.496 | KAN Test Kappa: 0.014 | \n",
      "Epoch 38/50 | MLP Train Acc: 0.651 | MLP F1: 0.651 | MLP Kappa: 0.298 | MLP Test Acc: 0.596 | MLP Test F1: 0.595 | MLP Test Kappa: 0.188 | \n",
      "Epoch 39/50 | KAN Train Acc: 0.650 | KAN F1: 0.627 | KAN Kappa: 0.274 | KAN Test Acc: 0.521 | KAN Test F1: 0.490 | KAN Test Kappa: 0.014 | \n",
      "Epoch 39/50 | MLP Train Acc: 0.651 | MLP F1: 0.651 | MLP Kappa: 0.297 | MLP Test Acc: 0.599 | MLP Test F1: 0.598 | MLP Test Kappa: 0.194 | \n",
      "Epoch 40/50 | KAN Train Acc: 0.652 | KAN F1: 0.628 | KAN Kappa: 0.277 | KAN Test Acc: 0.522 | KAN Test F1: 0.490 | KAN Test Kappa: 0.016 | \n",
      "Epoch 40/50 | MLP Train Acc: 0.650 | MLP F1: 0.650 | MLP Kappa: 0.296 | MLP Test Acc: 0.597 | MLP Test F1: 0.597 | MLP Test Kappa: 0.191 | \n",
      "Epoch 41/50 | KAN Train Acc: 0.651 | KAN F1: 0.634 | KAN Kappa: 0.280 | KAN Test Acc: 0.523 | KAN Test F1: 0.499 | KAN Test Kappa: 0.021 | \n",
      "Epoch 41/50 | MLP Train Acc: 0.650 | MLP F1: 0.650 | MLP Kappa: 0.296 | MLP Test Acc: 0.597 | MLP Test F1: 0.597 | MLP Test Kappa: 0.191 | \n",
      "Epoch 42/50 | KAN Train Acc: 0.653 | KAN F1: 0.641 | KAN Kappa: 0.288 | KAN Test Acc: 0.521 | KAN Test F1: 0.503 | KAN Test Kappa: 0.020 | \n",
      "Epoch 42/50 | MLP Train Acc: 0.651 | MLP F1: 0.651 | MLP Kappa: 0.298 | MLP Test Acc: 0.597 | MLP Test F1: 0.597 | MLP Test Kappa: 0.190 | \n",
      "Epoch 43/50 | KAN Train Acc: 0.654 | KAN F1: 0.643 | KAN Kappa: 0.290 | KAN Test Acc: 0.523 | KAN Test F1: 0.508 | KAN Test Kappa: 0.026 | \n",
      "Epoch 43/50 | MLP Train Acc: 0.650 | MLP F1: 0.650 | MLP Kappa: 0.296 | MLP Test Acc: 0.598 | MLP Test F1: 0.597 | MLP Test Kappa: 0.192 | \n",
      "Epoch 44/50 | KAN Train Acc: 0.657 | KAN F1: 0.648 | KAN Kappa: 0.298 | KAN Test Acc: 0.521 | KAN Test F1: 0.508 | KAN Test Kappa: 0.024 | \n",
      "Epoch 44/50 | MLP Train Acc: 0.650 | MLP F1: 0.650 | MLP Kappa: 0.296 | MLP Test Acc: 0.598 | MLP Test F1: 0.598 | MLP Test Kappa: 0.192 | \n",
      "Epoch 45/50 | KAN Train Acc: 0.660 | KAN F1: 0.655 | KAN Kappa: 0.307 | KAN Test Acc: 0.523 | KAN Test F1: 0.514 | KAN Test Kappa: 0.031 | \n",
      "Epoch 45/50 | MLP Train Acc: 0.652 | MLP F1: 0.652 | MLP Kappa: 0.301 | MLP Test Acc: 0.599 | MLP Test F1: 0.598 | MLP Test Kappa: 0.194 | \n",
      "Epoch 46/50 | KAN Train Acc: 0.661 | KAN F1: 0.655 | KAN Kappa: 0.309 | KAN Test Acc: 0.521 | KAN Test F1: 0.511 | KAN Test Kappa: 0.026 | \n",
      "Epoch 46/50 | MLP Train Acc: 0.652 | MLP F1: 0.652 | MLP Kappa: 0.300 | MLP Test Acc: 0.599 | MLP Test F1: 0.599 | MLP Test Kappa: 0.195 | \n",
      "Epoch 47/50 | KAN Train Acc: 0.663 | KAN F1: 0.658 | KAN Kappa: 0.313 | KAN Test Acc: 0.522 | KAN Test F1: 0.514 | KAN Test Kappa: 0.029 | \n",
      "Epoch 47/50 | MLP Train Acc: 0.653 | MLP F1: 0.653 | MLP Kappa: 0.302 | MLP Test Acc: 0.600 | MLP Test F1: 0.600 | MLP Test Kappa: 0.197 | \n",
      "Epoch 48/50 | KAN Train Acc: 0.663 | KAN F1: 0.655 | KAN Kappa: 0.311 | KAN Test Acc: 0.519 | KAN Test F1: 0.506 | KAN Test Kappa: 0.019 | \n",
      "Epoch 48/50 | MLP Train Acc: 0.652 | MLP F1: 0.652 | MLP Kappa: 0.301 | MLP Test Acc: 0.602 | MLP Test F1: 0.602 | MLP Test Kappa: 0.201 | \n",
      "Epoch 49/50 | KAN Train Acc: 0.661 | KAN F1: 0.652 | KAN Kappa: 0.306 | KAN Test Acc: 0.522 | KAN Test F1: 0.507 | KAN Test Kappa: 0.024 | \n",
      "Epoch 49/50 | MLP Train Acc: 0.653 | MLP F1: 0.652 | MLP Kappa: 0.301 | MLP Test Acc: 0.603 | MLP Test F1: 0.603 | MLP Test Kappa: 0.202 | \n",
      "Epoch 50/50 | KAN Train Acc: 0.663 | KAN F1: 0.651 | KAN Kappa: 0.308 | KAN Test Acc: 0.519 | KAN Test F1: 0.500 | KAN Test Kappa: 0.016 | \n",
      "Epoch 50/50 | MLP Train Acc: 0.653 | MLP F1: 0.653 | MLP Kappa: 0.303 | MLP Test Acc: 0.599 | MLP Test F1: 0.599 | MLP Test Kappa: 0.194 | \n"
     ]
    }
   ],
   "source": [
    "# ... (Rest of the code: accuracy, training loop)\n",
    "kan_model, mlp_model  = creating_models(X.shape[1], len(np.unique(y)))\n",
    "kan_optimizer = LBFGS(kan_model.parameters()) \n",
    "mlp_optimizer = LBFGS(mlp_model.parameters())  \n",
    "\n",
    "epochs = 50\n",
    "KAN_metrics_raw_hld= {}\n",
    "MLP_metrics_raw_hld= {}\n",
    "KAN_metrics_raw_hld['train']= []\n",
    "KAN_metrics_raw_hld['test']= []\n",
    "MLP_metrics_raw_hld['train']= []\n",
    "MLP_metrics_raw_hld['test']= []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    ### KAN Training\n",
    "    def kan_closure():  \n",
    "        kan_optimizer.zero_grad()\n",
    "        output = kan_model(X_train)\n",
    "        loss =  CrossEntropyLoss()(output, y_train)\n",
    "        loss.backward()\n",
    "        return loss\n",
    "    kan_optimizer.step(kan_closure)\n",
    "\n",
    "    def mlp_closure():\n",
    "        mlp_optimizer.zero_grad()\n",
    "        output = mlp_model(X_train)\n",
    "        loss = CrossEntropyLoss()(output, y_train)  # Using standard loss\n",
    "        loss.backward()\n",
    "        return loss\n",
    "    mlp_optimizer.step(mlp_closure)\n",
    "\n",
    "    kan_train_acc, kan_train_f1, kan_train_kappa = calculate_metrics(kan_model(X_train), y_train)\n",
    "    kan_test_acc, kan_test_f1, kan_test_kappa = calculate_metrics(kan_model(X_test), y_test) \n",
    "\n",
    "    mlp_train_acc, mlp_train_f1, mlp_train_kappa = calculate_metrics(mlp_model(X_train), y_train)\n",
    "    mlp_test_acc, mlp_test_f1, mlp_test_kappa = calculate_metrics(mlp_model(X_test), y_test)\n",
    "    print(f'Epoch {epoch+1}/{epochs} | '\n",
    "          f'KAN Train Acc: {kan_train_acc:.3f} | KAN F1: {kan_train_f1:.3f} | KAN Kappa: {kan_train_kappa:.3f} | '\n",
    "          f'KAN Test Acc: {kan_test_acc:.3f} | KAN Test F1: {kan_test_f1:.3f} | KAN Test Kappa: {kan_test_kappa:.3f} | ')\n",
    "    \n",
    "    print(f'Epoch {epoch+1}/{epochs} | '\n",
    "          f'MLP Train Acc: {mlp_train_acc:.3f} | MLP F1: {mlp_train_f1:.3f} | MLP Kappa: {mlp_train_kappa:.3f} | '\n",
    "          f'MLP Test Acc: {mlp_test_acc:.3f} | MLP Test F1: {mlp_test_f1:.3f} | MLP Test Kappa: {mlp_test_kappa:.3f} | ')\n",
    "    \n",
    "    KAN_metrics_raw_hld['train'].append([kan_train_acc, kan_train_f1, kan_train_kappa])\n",
    "    KAN_metrics_raw_hld['test'].append([kan_test_acc, kan_test_f1, kan_test_kappa ])\n",
    "    MLP_metrics_raw_hld['train'].append([mlp_train_acc, mlp_train_f1, mlp_train_kappa])\n",
    "    MLP_metrics_raw_hld['test'].append([mlp_test_acc, mlp_test_f1, mlp_test_kappa])\n",
    "\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting EDF parameters from C:\\Users\\JARS\\datasets\\MNE-eegbci-data\\files\\eegmmidb\\1.0.0\\S010\\S010R04.edf...\n",
      "EDF file detected\n",
      "Setting channel info structure...\n",
      "Creating raw.info structure...\n",
      "Reading 0 ... 19679  =      0.000 ...   122.994 secs...\n",
      "Filtering raw data in 1 contiguous segment\n",
      "Setting up band-pass filter from 7 - 35 Hz\n",
      "\n",
      "IIR filter parameters\n",
      "---------------------\n",
      "Butterworth bandpass zero-phase (two-pass forward and reverse) non-causal filter:\n",
      "- Filter order 16 (effective, after forward-backward)\n",
      "- Cutoffs at 7.00, 35.00 Hz: -6.02, -6.02 dB\n",
      "\n",
      "Used Annotations descriptions: ['T1', 'T2']\n",
      "Extracting EDF parameters from C:\\Users\\JARS\\datasets\\MNE-eegbci-data\\files\\eegmmidb\\1.0.0\\S010\\S010R08.edf...\n",
      "EDF file detected\n",
      "Setting channel info structure...\n",
      "Creating raw.info structure...\n",
      "Reading 0 ... 19679  =      0.000 ...   122.994 secs...\n",
      "Filtering raw data in 1 contiguous segment\n",
      "Setting up band-pass filter from 7 - 35 Hz\n",
      "\n",
      "IIR filter parameters\n",
      "---------------------\n",
      "Butterworth bandpass zero-phase (two-pass forward and reverse) non-causal filter:\n",
      "- Filter order 16 (effective, after forward-backward)\n",
      "- Cutoffs at 7.00, 35.00 Hz: -6.02, -6.02 dB\n",
      "\n",
      "Used Annotations descriptions: ['T1', 'T2']\n",
      "Extracting EDF parameters from C:\\Users\\JARS\\datasets\\MNE-eegbci-data\\files\\eegmmidb\\1.0.0\\S010\\S010R06.edf...\n",
      "EDF file detected\n",
      "Setting channel info structure...\n",
      "Creating raw.info structure...\n",
      "Reading 0 ... 19679  =      0.000 ...   122.994 secs...\n",
      "Filtering raw data in 1 contiguous segment\n",
      "Setting up band-pass filter from 7 - 35 Hz\n",
      "\n",
      "IIR filter parameters\n",
      "---------------------\n",
      "Butterworth bandpass zero-phase (two-pass forward and reverse) non-causal filter:\n",
      "- Filter order 16 (effective, after forward-backward)\n",
      "- Cutoffs at 7.00, 35.00 Hz: -6.02, -6.02 dB\n",
      "\n",
      "Used Annotations descriptions: ['T1', 'T2']\n",
      "Extracting EDF parameters from C:\\Users\\JARS\\datasets\\MNE-eegbci-data\\files\\eegmmidb\\1.0.0\\S010\\S010R10.edf...\n",
      "EDF file detected\n",
      "Setting channel info structure...\n",
      "Creating raw.info structure...\n",
      "Reading 0 ... 19679  =      0.000 ...   122.994 secs...\n",
      "Filtering raw data in 1 contiguous segment\n",
      "Setting up band-pass filter from 7 - 35 Hz\n",
      "\n",
      "IIR filter parameters\n",
      "---------------------\n",
      "Butterworth bandpass zero-phase (two-pass forward and reverse) non-causal filter:\n",
      "- Filter order 16 (effective, after forward-backward)\n",
      "- Cutoffs at 7.00, 35.00 Hz: -6.02, -6.02 dB\n",
      "\n",
      "Used Annotations descriptions: ['T1', 'T2']\n",
      "Not setting metadata\n",
      "60 matching events found\n",
      "No baseline correction applied\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "100%|██████████| 10/10 [00:00<00:00, 26.46it/s]\n"
     ]
    }
   ],
   "source": [
    "df= data_EEG(multiclass=True).to_data_frame().drop(columns=['time', 'epoch'])\n",
    "# Data Conversion (NumPy to PyTorch)\n",
    "selected_features = mrmr_classif(X=df.drop(columns=['condition']), y=df.condition, K=10)\n",
    "df_features = df[selected_features+['condition']]\n",
    "X = df.drop(columns=['condition']).values\n",
    "y = df.condition.values\n",
    "le = LabelEncoder()\n",
    "y = le.fit_transform(y)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)\n",
    "\n",
    "X_train = torch.from_numpy(X_train).double()\n",
    "X_test  = torch.from_numpy(X_test).double()\n",
    "y_train  = torch.from_numpy(y_train).long()\n",
    "y_test  = torch.from_numpy(y_test).long()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50 | KAN Train Acc: 0.335 | KAN F1: 0.275 | KAN Kappa: -0.006 | KAN Test Acc: 0.328 | KAN Test F1: 0.265 | KAN Test Kappa: -0.009 | \n",
      "Epoch 1/50 | MLP Train Acc: 0.364 | MLP F1: 0.276 | MLP Kappa: 0.018 | MLP Test Acc: 0.354 | MLP Test F1: 0.265 | MLP Test Kappa: 0.011 | \n",
      "Epoch 2/50 | KAN Train Acc: 0.360 | KAN F1: 0.252 | KAN Kappa: 0.009 | KAN Test Acc: 0.351 | KAN Test F1: 0.238 | KAN Test Kappa: 0.004 | \n",
      "Epoch 2/50 | MLP Train Acc: 0.382 | MLP F1: 0.306 | MLP Kappa: 0.049 | MLP Test Acc: 0.365 | MLP Test F1: 0.287 | MLP Test Kappa: 0.031 | \n",
      "Epoch 3/50 | KAN Train Acc: 0.362 | KAN F1: 0.260 | KAN Kappa: 0.013 | KAN Test Acc: 0.356 | KAN Test F1: 0.251 | KAN Test Kappa: 0.012 | \n",
      "Epoch 3/50 | MLP Train Acc: 0.390 | MLP F1: 0.316 | MLP Kappa: 0.062 | MLP Test Acc: 0.371 | MLP Test F1: 0.296 | MLP Test Kappa: 0.041 | \n",
      "Epoch 4/50 | KAN Train Acc: 0.331 | KAN F1: 0.326 | KAN Kappa: -0.010 | KAN Test Acc: 0.330 | KAN Test F1: 0.324 | KAN Test Kappa: -0.008 | \n",
      "Epoch 4/50 | MLP Train Acc: 0.394 | MLP F1: 0.321 | MLP Kappa: 0.068 | MLP Test Acc: 0.378 | MLP Test F1: 0.304 | MLP Test Kappa: 0.052 | \n",
      "Epoch 5/50 | KAN Train Acc: 0.372 | KAN F1: 0.291 | KAN Kappa: 0.032 | KAN Test Acc: 0.359 | KAN Test F1: 0.275 | KAN Test Kappa: 0.020 | \n",
      "Epoch 5/50 | MLP Train Acc: 0.397 | MLP F1: 0.326 | MLP Kappa: 0.074 | MLP Test Acc: 0.384 | MLP Test F1: 0.312 | MLP Test Kappa: 0.061 | \n",
      "Epoch 6/50 | KAN Train Acc: 0.373 | KAN F1: 0.303 | KAN Kappa: 0.034 | KAN Test Acc: 0.360 | KAN Test F1: 0.288 | KAN Test Kappa: 0.023 | \n",
      "Epoch 6/50 | MLP Train Acc: 0.396 | MLP F1: 0.330 | MLP Kappa: 0.073 | MLP Test Acc: 0.386 | MLP Test F1: 0.319 | MLP Test Kappa: 0.066 | \n",
      "Epoch 7/50 | KAN Train Acc: 0.376 | KAN F1: 0.305 | KAN Kappa: 0.038 | KAN Test Acc: 0.361 | KAN Test F1: 0.290 | KAN Test Kappa: 0.024 | \n",
      "Epoch 7/50 | MLP Train Acc: 0.398 | MLP F1: 0.333 | MLP Kappa: 0.076 | MLP Test Acc: 0.389 | MLP Test F1: 0.322 | MLP Test Kappa: 0.070 | \n",
      "Epoch 8/50 | KAN Train Acc: 0.378 | KAN F1: 0.310 | KAN Kappa: 0.041 | KAN Test Acc: 0.362 | KAN Test F1: 0.293 | KAN Test Kappa: 0.026 | \n",
      "Epoch 8/50 | MLP Train Acc: 0.398 | MLP F1: 0.342 | MLP Kappa: 0.077 | MLP Test Acc: 0.385 | MLP Test F1: 0.326 | MLP Test Kappa: 0.064 | \n",
      "Epoch 9/50 | KAN Train Acc: 0.379 | KAN F1: 0.326 | KAN Kappa: 0.046 | KAN Test Acc: 0.363 | KAN Test F1: 0.308 | KAN Test Kappa: 0.029 | \n",
      "Epoch 9/50 | MLP Train Acc: 0.401 | MLP F1: 0.348 | MLP Kappa: 0.082 | MLP Test Acc: 0.386 | MLP Test F1: 0.330 | MLP Test Kappa: 0.066 | \n",
      "Epoch 10/50 | KAN Train Acc: 0.381 | KAN F1: 0.328 | KAN Kappa: 0.049 | KAN Test Acc: 0.362 | KAN Test F1: 0.306 | KAN Test Kappa: 0.027 | \n",
      "Epoch 10/50 | MLP Train Acc: 0.402 | MLP F1: 0.347 | MLP Kappa: 0.083 | MLP Test Acc: 0.388 | MLP Test F1: 0.330 | MLP Test Kappa: 0.070 | \n",
      "Epoch 11/50 | KAN Train Acc: 0.383 | KAN F1: 0.337 | KAN Kappa: 0.054 | KAN Test Acc: 0.361 | KAN Test F1: 0.313 | KAN Test Kappa: 0.026 | \n",
      "Epoch 11/50 | MLP Train Acc: 0.401 | MLP F1: 0.349 | MLP Kappa: 0.083 | MLP Test Acc: 0.389 | MLP Test F1: 0.334 | MLP Test Kappa: 0.071 | \n",
      "Epoch 12/50 | KAN Train Acc: 0.386 | KAN F1: 0.340 | KAN Kappa: 0.058 | KAN Test Acc: 0.364 | KAN Test F1: 0.316 | KAN Test Kappa: 0.031 | \n",
      "Epoch 12/50 | MLP Train Acc: 0.403 | MLP F1: 0.352 | MLP Kappa: 0.085 | MLP Test Acc: 0.392 | MLP Test F1: 0.340 | MLP Test Kappa: 0.076 | \n",
      "Epoch 13/50 | KAN Train Acc: 0.390 | KAN F1: 0.359 | KAN Kappa: 0.068 | KAN Test Acc: 0.359 | KAN Test F1: 0.327 | KAN Test Kappa: 0.027 | \n",
      "Epoch 13/50 | MLP Train Acc: 0.403 | MLP F1: 0.356 | MLP Kappa: 0.086 | MLP Test Acc: 0.388 | MLP Test F1: 0.337 | MLP Test Kappa: 0.069 | \n",
      "Epoch 14/50 | KAN Train Acc: 0.391 | KAN F1: 0.365 | KAN Kappa: 0.071 | KAN Test Acc: 0.356 | KAN Test F1: 0.328 | KAN Test Kappa: 0.023 | \n",
      "Epoch 14/50 | MLP Train Acc: 0.403 | MLP F1: 0.356 | MLP Kappa: 0.086 | MLP Test Acc: 0.392 | MLP Test F1: 0.341 | MLP Test Kappa: 0.075 | \n",
      "Epoch 15/50 | KAN Train Acc: 0.390 | KAN F1: 0.368 | KAN Kappa: 0.071 | KAN Test Acc: 0.358 | KAN Test F1: 0.334 | KAN Test Kappa: 0.027 | \n",
      "Epoch 15/50 | MLP Train Acc: 0.404 | MLP F1: 0.356 | MLP Kappa: 0.087 | MLP Test Acc: 0.391 | MLP Test F1: 0.340 | MLP Test Kappa: 0.074 | \n",
      "Epoch 16/50 | KAN Train Acc: 0.396 | KAN F1: 0.369 | KAN Kappa: 0.078 | KAN Test Acc: 0.360 | KAN Test F1: 0.333 | KAN Test Kappa: 0.029 | \n",
      "Epoch 16/50 | MLP Train Acc: 0.404 | MLP F1: 0.356 | MLP Kappa: 0.087 | MLP Test Acc: 0.390 | MLP Test F1: 0.339 | MLP Test Kappa: 0.072 | \n",
      "Epoch 17/50 | KAN Train Acc: 0.392 | KAN F1: 0.365 | KAN Kappa: 0.072 | KAN Test Acc: 0.356 | KAN Test F1: 0.326 | KAN Test Kappa: 0.022 | \n",
      "Epoch 17/50 | MLP Train Acc: 0.405 | MLP F1: 0.356 | MLP Kappa: 0.089 | MLP Test Acc: 0.386 | MLP Test F1: 0.336 | MLP Test Kappa: 0.067 | \n",
      "Epoch 18/50 | KAN Train Acc: 0.399 | KAN F1: 0.370 | KAN Kappa: 0.082 | KAN Test Acc: 0.357 | KAN Test F1: 0.326 | KAN Test Kappa: 0.023 | \n",
      "Epoch 18/50 | MLP Train Acc: 0.404 | MLP F1: 0.355 | MLP Kappa: 0.087 | MLP Test Acc: 0.389 | MLP Test F1: 0.338 | MLP Test Kappa: 0.071 | \n",
      "Epoch 19/50 | KAN Train Acc: 0.400 | KAN F1: 0.375 | KAN Kappa: 0.085 | KAN Test Acc: 0.355 | KAN Test F1: 0.328 | KAN Test Kappa: 0.021 | \n",
      "Epoch 19/50 | MLP Train Acc: 0.405 | MLP F1: 0.357 | MLP Kappa: 0.088 | MLP Test Acc: 0.387 | MLP Test F1: 0.336 | MLP Test Kappa: 0.068 | \n",
      "Epoch 20/50 | KAN Train Acc: 0.402 | KAN F1: 0.379 | KAN Kappa: 0.088 | KAN Test Acc: 0.354 | KAN Test F1: 0.329 | KAN Test Kappa: 0.020 | \n",
      "Epoch 20/50 | MLP Train Acc: 0.403 | MLP F1: 0.355 | MLP Kappa: 0.086 | MLP Test Acc: 0.389 | MLP Test F1: 0.341 | MLP Test Kappa: 0.072 | \n",
      "Epoch 21/50 | KAN Train Acc: 0.405 | KAN F1: 0.380 | KAN Kappa: 0.094 | KAN Test Acc: 0.352 | KAN Test F1: 0.325 | KAN Test Kappa: 0.018 | \n",
      "Epoch 21/50 | MLP Train Acc: 0.404 | MLP F1: 0.357 | MLP Kappa: 0.088 | MLP Test Acc: 0.387 | MLP Test F1: 0.339 | MLP Test Kappa: 0.069 | \n",
      "Epoch 22/50 | KAN Train Acc: 0.408 | KAN F1: 0.391 | KAN Kappa: 0.100 | KAN Test Acc: 0.345 | KAN Test F1: 0.327 | KAN Test Kappa: 0.008 | \n",
      "Epoch 22/50 | MLP Train Acc: 0.406 | MLP F1: 0.358 | MLP Kappa: 0.090 | MLP Test Acc: 0.388 | MLP Test F1: 0.340 | MLP Test Kappa: 0.070 | \n",
      "Epoch 23/50 | KAN Train Acc: 0.412 | KAN F1: 0.397 | KAN Kappa: 0.107 | KAN Test Acc: 0.347 | KAN Test F1: 0.331 | KAN Test Kappa: 0.011 | \n",
      "Epoch 23/50 | MLP Train Acc: 0.406 | MLP F1: 0.360 | MLP Kappa: 0.091 | MLP Test Acc: 0.388 | MLP Test F1: 0.343 | MLP Test Kappa: 0.071 | \n",
      "Epoch 24/50 | KAN Train Acc: 0.418 | KAN F1: 0.402 | KAN Kappa: 0.115 | KAN Test Acc: 0.345 | KAN Test F1: 0.328 | KAN Test Kappa: 0.009 | \n",
      "Epoch 24/50 | MLP Train Acc: 0.408 | MLP F1: 0.361 | MLP Kappa: 0.094 | MLP Test Acc: 0.391 | MLP Test F1: 0.345 | MLP Test Kappa: 0.075 | \n",
      "Epoch 25/50 | KAN Train Acc: 0.420 | KAN F1: 0.406 | KAN Kappa: 0.118 | KAN Test Acc: 0.351 | KAN Test F1: 0.336 | KAN Test Kappa: 0.018 | \n",
      "Epoch 25/50 | MLP Train Acc: 0.406 | MLP F1: 0.362 | MLP Kappa: 0.092 | MLP Test Acc: 0.393 | MLP Test F1: 0.347 | MLP Test Kappa: 0.078 | \n",
      "Epoch 26/50 | KAN Train Acc: 0.421 | KAN F1: 0.405 | KAN Kappa: 0.120 | KAN Test Acc: 0.350 | KAN Test F1: 0.332 | KAN Test Kappa: 0.015 | \n",
      "Epoch 26/50 | MLP Train Acc: 0.408 | MLP F1: 0.363 | MLP Kappa: 0.094 | MLP Test Acc: 0.393 | MLP Test F1: 0.347 | MLP Test Kappa: 0.078 | \n",
      "Epoch 27/50 | KAN Train Acc: 0.424 | KAN F1: 0.411 | KAN Kappa: 0.125 | KAN Test Acc: 0.348 | KAN Test F1: 0.332 | KAN Test Kappa: 0.014 | \n",
      "Epoch 27/50 | MLP Train Acc: 0.406 | MLP F1: 0.361 | MLP Kappa: 0.091 | MLP Test Acc: 0.394 | MLP Test F1: 0.348 | MLP Test Kappa: 0.080 | \n",
      "Epoch 28/50 | KAN Train Acc: 0.423 | KAN F1: 0.414 | KAN Kappa: 0.126 | KAN Test Acc: 0.351 | KAN Test F1: 0.339 | KAN Test Kappa: 0.019 | \n",
      "Epoch 28/50 | MLP Train Acc: 0.406 | MLP F1: 0.361 | MLP Kappa: 0.092 | MLP Test Acc: 0.392 | MLP Test F1: 0.347 | MLP Test Kappa: 0.076 | \n",
      "Epoch 29/50 | KAN Train Acc: 0.428 | KAN F1: 0.422 | KAN Kappa: 0.135 | KAN Test Acc: 0.349 | KAN Test F1: 0.341 | KAN Test Kappa: 0.017 | \n",
      "Epoch 29/50 | MLP Train Acc: 0.406 | MLP F1: 0.361 | MLP Kappa: 0.091 | MLP Test Acc: 0.391 | MLP Test F1: 0.346 | MLP Test Kappa: 0.075 | \n",
      "Epoch 30/50 | KAN Train Acc: 0.431 | KAN F1: 0.422 | KAN Kappa: 0.138 | KAN Test Acc: 0.349 | KAN Test F1: 0.337 | KAN Test Kappa: 0.016 | \n",
      "Epoch 30/50 | MLP Train Acc: 0.407 | MLP F1: 0.361 | MLP Kappa: 0.092 | MLP Test Acc: 0.389 | MLP Test F1: 0.343 | MLP Test Kappa: 0.072 | \n",
      "Epoch 31/50 | KAN Train Acc: 0.436 | KAN F1: 0.426 | KAN Kappa: 0.144 | KAN Test Acc: 0.350 | KAN Test F1: 0.338 | KAN Test Kappa: 0.017 | \n",
      "Epoch 31/50 | MLP Train Acc: 0.405 | MLP F1: 0.360 | MLP Kappa: 0.090 | MLP Test Acc: 0.388 | MLP Test F1: 0.343 | MLP Test Kappa: 0.070 | \n",
      "Epoch 32/50 | KAN Train Acc: 0.438 | KAN F1: 0.429 | KAN Kappa: 0.148 | KAN Test Acc: 0.347 | KAN Test F1: 0.337 | KAN Test Kappa: 0.014 | \n",
      "Epoch 32/50 | MLP Train Acc: 0.405 | MLP F1: 0.360 | MLP Kappa: 0.089 | MLP Test Acc: 0.388 | MLP Test F1: 0.343 | MLP Test Kappa: 0.071 | \n",
      "Epoch 33/50 | KAN Train Acc: 0.440 | KAN F1: 0.432 | KAN Kappa: 0.151 | KAN Test Acc: 0.347 | KAN Test F1: 0.337 | KAN Test Kappa: 0.014 | \n",
      "Epoch 33/50 | MLP Train Acc: 0.405 | MLP F1: 0.362 | MLP Kappa: 0.090 | MLP Test Acc: 0.388 | MLP Test F1: 0.344 | MLP Test Kappa: 0.071 | \n",
      "Epoch 34/50 | KAN Train Acc: 0.442 | KAN F1: 0.435 | KAN Kappa: 0.155 | KAN Test Acc: 0.348 | KAN Test F1: 0.339 | KAN Test Kappa: 0.016 | \n",
      "Epoch 34/50 | MLP Train Acc: 0.405 | MLP F1: 0.362 | MLP Kappa: 0.090 | MLP Test Acc: 0.388 | MLP Test F1: 0.344 | MLP Test Kappa: 0.071 | \n",
      "Epoch 35/50 | KAN Train Acc: 0.444 | KAN F1: 0.436 | KAN Kappa: 0.158 | KAN Test Acc: 0.350 | KAN Test F1: 0.340 | KAN Test Kappa: 0.018 | \n",
      "Epoch 35/50 | MLP Train Acc: 0.406 | MLP F1: 0.361 | MLP Kappa: 0.091 | MLP Test Acc: 0.390 | MLP Test F1: 0.346 | MLP Test Kappa: 0.073 | \n",
      "Epoch 36/50 | KAN Train Acc: 0.443 | KAN F1: 0.436 | KAN Kappa: 0.157 | KAN Test Acc: 0.347 | KAN Test F1: 0.339 | KAN Test Kappa: 0.015 | \n",
      "Epoch 36/50 | MLP Train Acc: 0.406 | MLP F1: 0.362 | MLP Kappa: 0.091 | MLP Test Acc: 0.389 | MLP Test F1: 0.346 | MLP Test Kappa: 0.072 | \n",
      "Epoch 37/50 | KAN Train Acc: 0.448 | KAN F1: 0.443 | KAN Kappa: 0.165 | KAN Test Acc: 0.345 | KAN Test F1: 0.339 | KAN Test Kappa: 0.013 | \n",
      "Epoch 37/50 | MLP Train Acc: 0.406 | MLP F1: 0.362 | MLP Kappa: 0.092 | MLP Test Acc: 0.388 | MLP Test F1: 0.344 | MLP Test Kappa: 0.071 | \n",
      "Epoch 38/50 | KAN Train Acc: 0.449 | KAN F1: 0.445 | KAN Kappa: 0.168 | KAN Test Acc: 0.341 | KAN Test F1: 0.336 | KAN Test Kappa: 0.006 | \n",
      "Epoch 38/50 | MLP Train Acc: 0.407 | MLP F1: 0.364 | MLP Kappa: 0.093 | MLP Test Acc: 0.389 | MLP Test F1: 0.345 | MLP Test Kappa: 0.072 | \n",
      "Epoch 39/50 | KAN Train Acc: 0.452 | KAN F1: 0.449 | KAN Kappa: 0.173 | KAN Test Acc: 0.343 | KAN Test F1: 0.338 | KAN Test Kappa: 0.011 | \n",
      "Epoch 39/50 | MLP Train Acc: 0.407 | MLP F1: 0.363 | MLP Kappa: 0.093 | MLP Test Acc: 0.388 | MLP Test F1: 0.344 | MLP Test Kappa: 0.071 | \n",
      "Epoch 40/50 | KAN Train Acc: 0.452 | KAN F1: 0.448 | KAN Kappa: 0.172 | KAN Test Acc: 0.345 | KAN Test F1: 0.339 | KAN Test Kappa: 0.013 | \n",
      "Epoch 40/50 | MLP Train Acc: 0.407 | MLP F1: 0.363 | MLP Kappa: 0.093 | MLP Test Acc: 0.390 | MLP Test F1: 0.346 | MLP Test Kappa: 0.074 | \n",
      "Epoch 41/50 | KAN Train Acc: 0.453 | KAN F1: 0.450 | KAN Kappa: 0.174 | KAN Test Acc: 0.343 | KAN Test F1: 0.338 | KAN Test Kappa: 0.010 | \n",
      "Epoch 41/50 | MLP Train Acc: 0.406 | MLP F1: 0.362 | MLP Kappa: 0.092 | MLP Test Acc: 0.390 | MLP Test F1: 0.345 | MLP Test Kappa: 0.073 | \n",
      "Epoch 42/50 | KAN Train Acc: 0.457 | KAN F1: 0.453 | KAN Kappa: 0.179 | KAN Test Acc: 0.342 | KAN Test F1: 0.337 | KAN Test Kappa: 0.008 | \n",
      "Epoch 42/50 | MLP Train Acc: 0.406 | MLP F1: 0.361 | MLP Kappa: 0.091 | MLP Test Acc: 0.390 | MLP Test F1: 0.346 | MLP Test Kappa: 0.074 | \n",
      "Epoch 43/50 | KAN Train Acc: 0.458 | KAN F1: 0.454 | KAN Kappa: 0.181 | KAN Test Acc: 0.343 | KAN Test F1: 0.336 | KAN Test Kappa: 0.009 | \n",
      "Epoch 43/50 | MLP Train Acc: 0.407 | MLP F1: 0.362 | MLP Kappa: 0.092 | MLP Test Acc: 0.390 | MLP Test F1: 0.347 | MLP Test Kappa: 0.074 | \n",
      "Epoch 44/50 | KAN Train Acc: 0.461 | KAN F1: 0.457 | KAN Kappa: 0.186 | KAN Test Acc: 0.341 | KAN Test F1: 0.336 | KAN Test Kappa: 0.008 | \n",
      "Epoch 44/50 | MLP Train Acc: 0.406 | MLP F1: 0.362 | MLP Kappa: 0.092 | MLP Test Acc: 0.390 | MLP Test F1: 0.346 | MLP Test Kappa: 0.074 | \n",
      "Epoch 45/50 | KAN Train Acc: 0.460 | KAN F1: 0.456 | KAN Kappa: 0.184 | KAN Test Acc: 0.338 | KAN Test F1: 0.333 | KAN Test Kappa: 0.003 | \n",
      "Epoch 45/50 | MLP Train Acc: 0.406 | MLP F1: 0.362 | MLP Kappa: 0.092 | MLP Test Acc: 0.389 | MLP Test F1: 0.346 | MLP Test Kappa: 0.072 | \n",
      "Epoch 46/50 | KAN Train Acc: 0.464 | KAN F1: 0.461 | KAN Kappa: 0.191 | KAN Test Acc: 0.339 | KAN Test F1: 0.335 | KAN Test Kappa: 0.005 | \n",
      "Epoch 46/50 | MLP Train Acc: 0.406 | MLP F1: 0.362 | MLP Kappa: 0.091 | MLP Test Acc: 0.389 | MLP Test F1: 0.346 | MLP Test Kappa: 0.072 | \n",
      "Epoch 47/50 | KAN Train Acc: 0.465 | KAN F1: 0.461 | KAN Kappa: 0.192 | KAN Test Acc: 0.340 | KAN Test F1: 0.335 | KAN Test Kappa: 0.005 | \n",
      "Epoch 47/50 | MLP Train Acc: 0.406 | MLP F1: 0.362 | MLP Kappa: 0.092 | MLP Test Acc: 0.390 | MLP Test F1: 0.347 | MLP Test Kappa: 0.073 | \n",
      "Epoch 48/50 | KAN Train Acc: 0.468 | KAN F1: 0.465 | KAN Kappa: 0.196 | KAN Test Acc: 0.340 | KAN Test F1: 0.336 | KAN Test Kappa: 0.006 | \n",
      "Epoch 48/50 | MLP Train Acc: 0.407 | MLP F1: 0.363 | MLP Kappa: 0.093 | MLP Test Acc: 0.390 | MLP Test F1: 0.347 | MLP Test Kappa: 0.074 | \n",
      "Epoch 49/50 | KAN Train Acc: 0.471 | KAN F1: 0.468 | KAN Kappa: 0.201 | KAN Test Acc: 0.345 | KAN Test F1: 0.340 | KAN Test Kappa: 0.012 | \n",
      "Epoch 49/50 | MLP Train Acc: 0.406 | MLP F1: 0.362 | MLP Kappa: 0.091 | MLP Test Acc: 0.388 | MLP Test F1: 0.345 | MLP Test Kappa: 0.071 | \n",
      "Epoch 50/50 | KAN Train Acc: 0.469 | KAN F1: 0.466 | KAN Kappa: 0.199 | KAN Test Acc: 0.339 | KAN Test F1: 0.335 | KAN Test Kappa: 0.005 | \n",
      "Epoch 50/50 | MLP Train Acc: 0.408 | MLP F1: 0.363 | MLP Kappa: 0.094 | MLP Test Acc: 0.386 | MLP Test F1: 0.342 | MLP Test Kappa: 0.068 | \n"
     ]
    }
   ],
   "source": [
    "# ... (Rest of the code: accuracy, training loop)\n",
    "kan_model, mlp_model  = creating_models(X.shape[1], len(np.unique(y)))\n",
    "kan_optimizer = LBFGS(kan_model.parameters()) \n",
    "mlp_optimizer = LBFGS(mlp_model.parameters())  \n",
    "\n",
    "epochs = 50\n",
    "KAN_metrics_raw_hld_mc= {}\n",
    "MLP_metrics_raw_hld_mc= {}\n",
    "KAN_metrics_raw_hld_mc['train']= []\n",
    "KAN_metrics_raw_hld_mc['test']= []\n",
    "MLP_metrics_raw_hld_mc['train']= []\n",
    "MLP_metrics_raw_hld_mc['test']= []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    ### KAN Training\n",
    "    def kan_closure():  \n",
    "        kan_optimizer.zero_grad()\n",
    "        output = kan_model(X_train)\n",
    "        loss =  CrossEntropyLoss()(output, y_train)\n",
    "        loss.backward()\n",
    "        return loss\n",
    "    kan_optimizer.step(kan_closure)\n",
    "\n",
    "    def mlp_closure():\n",
    "        mlp_optimizer.zero_grad()\n",
    "        output = mlp_model(X_train)\n",
    "        loss = CrossEntropyLoss()(output, y_train)  # Using standard loss\n",
    "        loss.backward()\n",
    "        return loss\n",
    "    mlp_optimizer.step(mlp_closure)\n",
    "\n",
    "    kan_train_acc, kan_train_f1, kan_train_kappa = calculate_metrics(kan_model(X_train), y_train)\n",
    "    kan_test_acc, kan_test_f1, kan_test_kappa = calculate_metrics(kan_model(X_test), y_test) \n",
    "\n",
    "    mlp_train_acc, mlp_train_f1, mlp_train_kappa = calculate_metrics(mlp_model(X_train), y_train)\n",
    "    mlp_test_acc, mlp_test_f1, mlp_test_kappa = calculate_metrics(mlp_model(X_test), y_test)\n",
    "    print(f'Epoch {epoch+1}/{epochs} | '\n",
    "          f'KAN Train Acc: {kan_train_acc:.3f} | KAN F1: {kan_train_f1:.3f} | KAN Kappa: {kan_train_kappa:.3f} | '\n",
    "          f'KAN Test Acc: {kan_test_acc:.3f} | KAN Test F1: {kan_test_f1:.3f} | KAN Test Kappa: {kan_test_kappa:.3f} | ')\n",
    "    \n",
    "    print(f'Epoch {epoch+1}/{epochs} | '\n",
    "          f'MLP Train Acc: {mlp_train_acc:.3f} | MLP F1: {mlp_train_f1:.3f} | MLP Kappa: {mlp_train_kappa:.3f} | '\n",
    "          f'MLP Test Acc: {mlp_test_acc:.3f} | MLP Test F1: {mlp_test_f1:.3f} | MLP Test Kappa: {mlp_test_kappa:.3f} | ')\n",
    "    \n",
    "    KAN_metrics_raw_hld_mc['train'].append([kan_train_acc, kan_train_f1, kan_train_kappa])\n",
    "    KAN_metrics_raw_hld_mc['test'].append([kan_test_acc, kan_test_f1, kan_test_kappa ])\n",
    "    MLP_metrics_raw_hld_mc['train'].append([mlp_train_acc, mlp_train_f1, mlp_train_kappa])\n",
    "    MLP_metrics_raw_hld_mc['test'].append([mlp_test_acc, mlp_test_f1, mlp_test_kappa])\n",
    "\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df= data_EEG(multiclass=True).to_data_frame().drop(columns=['time', 'epoch'])\n",
    "# Data Conversion (NumPy to PyTorch)\n",
    "selected_features = mrmr_classif(X=df.drop(columns=['condition']), y=df.condition, K=10)\n",
    "df_features = df[selected_features+['condition']]\n",
    "X = df_features.drop(columns=['condition']).values\n",
    "y = df_features.condition.values\n",
    "le = LabelEncoder()\n",
    "y = le.fit_transform(y)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)\n",
    "\n",
    "X_train = torch.from_numpy(X_train).double()\n",
    "X_test  = torch.from_numpy(X_test).double()\n",
    "y_train  = torch.from_numpy(y_train).long()\n",
    "y_test  = torch.from_numpy(y_test).long()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#CSP Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = data_EEG()\n",
    "# Data Conversion (NumPy to PyTorch)\n",
    "X = np.reshape(X, [np.shape(X)[0]*np.shape(X)[1], np.shape(X)[2]])\n",
    "y = np.reshape(y, [np.shape(y)[0]*np.shape(y)[1]])\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)\n",
    "\n",
    "# Convert to PyTorch tensors\n",
    "X_train = torch.from_numpy(X_train).double() \n",
    "X_test = torch.from_numpy(X_test).double()\n",
    "y_train = torch.from_numpy(y_train).long()  \n",
    "y_test = torch.from_numpy(y_test).long()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# --- KAN Model --- \n",
    "model = KAN(width=[X_train.shape[1], 2],  # Input features, 2 output classes\n",
    "            grid=5,          \n",
    "            k=3)             \n",
    "\n",
    "# Ensure double-precision tensors\n",
    "for param in model.parameters():\n",
    "    param.data = param.data.double()\n",
    "\n",
    "optimizer = LBFGS(model.parameters()) \n",
    "\n",
    "\n",
    "\n",
    "# --- MLP Model --- \n",
    "mlp_model = Sequential(\n",
    "      Linear(X_train.shape[1], 8).double(),  # Input features \n",
    "      ReLU(),\n",
    "      Linear(8, 2).double()  # 2 output classes \n",
    ")\n",
    "\n",
    "# Ensure double-precision tensors\n",
    "for param in mlp_model.parameters():\n",
    "    param.data = param.data.double()\n",
    "\n",
    "mlp_optimizer = LBFGS(mlp_model.parameters())  \n",
    "\n",
    "# ... (Rest of the code: accuracy, training loop)\n",
    "epochs = 20\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    ### KAN Training\n",
    "    def kan_closure():  \n",
    "        optimizer.zero_grad()\n",
    "        output = model(X_train)\n",
    "        loss =  CrossEntropyLoss()(output, y_train)\n",
    "        loss.backward()\n",
    "        return loss\n",
    "    optimizer.step(kan_closure)\n",
    "\n",
    "    ### MLP Training\n",
    "    def mlp_closure():\n",
    "        mlp_optimizer.zero_grad()\n",
    "        output = mlp_model(X_train)\n",
    "        loss = CrossEntropyLoss()(output, y_train)  # Using standard loss\n",
    "        loss.backward()\n",
    "        return loss\n",
    "    mlp_optimizer.step(mlp_closure)\n",
    "\n",
    "    kan_train_acc, kan_train_f1, kan_train_kappa = calculate_metrics(model(X_train), y_train)\n",
    "    kan_test_acc, kan_test_f1, kan_test_kappa = calculate_metrics(model(X_test), y_test) \n",
    "\n",
    "    mlp_train_acc, mlp_train_f1, mlp_train_kappa = calculate_metrics(mlp_model(X_train), y_train)\n",
    "    mlp_test_acc, mlp_test_f1, mlp_test_kappa = calculate_metrics(mlp_model(X_test), y_test)\n",
    "\n",
    "\n",
    "    print(f'Epoch {epoch+1}/{epochs} | '\n",
    "          f'KAN Train Acc: {kan_train_acc:.3f} | KAN F1: {kan_train_f1:.3f} | KAN Kappa: {kan_train_kappa:.3f} | '\n",
    "          f'MLP Train Acc: {mlp_train_acc:.3f} | MLP F1: {mlp_train_f1:.3f} | MLP Kappa: {mlp_train_kappa:.3f}') "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#RAW - CV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ... (Rest of the code: accuracy, training loop)\n",
    "epochs = 50\n",
    "kf = KFold(n_splits=10, shuffle=True, random_state=42)  # 5-fold cross-validation\n",
    "\n",
    "from mrmr import mrmr_classif\n",
    "\n",
    "df= data_EEG(raw_data=True)\n",
    "# Data Conversion (NumPy to PyTorch)\n",
    "selected_features = mrmr_classif(X=df.drop(columns=['events_id']), y=df.events_id, K=10)\n",
    "df_features = df[selected_features+['events_id']]\n",
    "\n",
    "# Data Conversion (NumPy to PyTorch)\n",
    "X = df_features.drop(columns=['events_id']).values\n",
    "y = df_features.events_id.values\n",
    "\n",
    "X = torch.from_numpy(X).double()   # Convert to PyTorch Tensor \n",
    "y = torch.from_numpy(y).long()  # Convert to PyTorch tensor \n",
    "\n",
    "\n",
    "\n",
    "optimizer = LBFGS(model.parameters()) \n",
    "\n",
    "\n",
    "\n",
    "# --- MLP Model --- \n",
    "\n",
    "mlp_optimizer = LBFGS(mlp_model.parameters())  \n",
    "\n",
    "for fold, (train_index, test_index) in enumerate(kf.split(X)):\n",
    "    # Prepare data for this fold\n",
    "    X_train, X_test = X[train_index], X[test_index]\n",
    "    y_train, y_test = y[train_index], y[test_index]\n",
    "\n",
    "\n",
    "# ... (Rest of the code: accuracy, training loop)\n",
    "    epochs = 20\n",
    "\n",
    "    for epoch in range(epochs):\n",
    "        ### KAN Training\n",
    "        def kan_closure():  \n",
    "            optimizer.zero_grad()\n",
    "            output = model(X_train)\n",
    "            loss =  CrossEntropyLoss()(output, y_train)\n",
    "            loss.backward()\n",
    "            return loss\n",
    "        optimizer.step(kan_closure)\n",
    "\n",
    "        ### MLP Training\n",
    "        def mlp_closure():\n",
    "            mlp_optimizer.zero_grad()\n",
    "            output = mlp_model(X_train)\n",
    "            loss = CrossEntropyLoss()(output, y_train)  # Using standard loss\n",
    "            loss.backward()\n",
    "            return loss\n",
    "        mlp_optimizer.step(mlp_closure)\n",
    "\n",
    "        kan_train_acc, kan_train_f1, kan_train_kappa = calculate_metrics(model(X_train), y_train)\n",
    "        kan_test_acc, kan_test_f1, kan_test_kappa = calculate_metrics(model(X_test), y_test) \n",
    "\n",
    "        mlp_train_acc, mlp_train_f1, mlp_train_kappa = calculate_metrics(mlp_model(X_train), y_train)\n",
    "        mlp_test_acc, mlp_test_f1, mlp_test_kappa = calculate_metrics(mlp_model(X_test), y_test)\n",
    "\n",
    "        print(f'KFOLD {fold}')\n",
    "        print(f'Epoch {epoch+1}/{epochs} | '\n",
    "            f'KAN Train Acc: {kan_train_acc:.3f} | KAN F1: {kan_train_f1:.3f} | KAN Kappa: {kan_train_kappa:.3f} | '\n",
    "            f'MLP Train Acc: {mlp_train_acc:.3f} | MLP F1: {mlp_train_f1:.3f} | MLP Kappa: {mlp_train_kappa:.3f}') "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#CSP - CV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# ... (Rest of the code: accuracy, training loop)\n",
    "epochs = 50\n",
    "kf = KFold(n_splits=10, shuffle=True, random_state=42)  # 5-fold cross-validation\n",
    "\n",
    "X, y = data_EEG()\n",
    "X = np.reshape(X, [np.shape(X)[0]*np.shape(X)[1], np.shape(X)[2]])\n",
    "y = np.reshape(y, [np.shape(y)[0]*np.shape(y)[1]])\n",
    "\n",
    "X = torch.from_numpy(X).double()   # Convert to PyTorch Tensor \n",
    "y = torch.from_numpy(y).long()  # Convert to PyTorch tensor \n",
    "\n",
    "model = KAN(width=[X.shape[1], 2],  # Input features, 2 output classes\n",
    "            grid=5,          \n",
    "            k=3)             \n",
    "\n",
    "# Ensure double-precision tensors\n",
    "for param in model.parameters():\n",
    "    param.data = param.data.double()\n",
    "\n",
    "optimizer = LBFGS(model.parameters()) \n",
    "\n",
    "\n",
    "\n",
    "# --- MLP Model --- \n",
    "mlp_model = Sequential(\n",
    "      Linear(X.shape[1], 8).double(),  # Input features \n",
    "      ReLU(),\n",
    "      Linear(8, 2).double()  # 2 output classes \n",
    ")\n",
    "\n",
    "# Ensure double-precision tensors\n",
    "for param in mlp_model.parameters():\n",
    "    param.data = param.data.double()\n",
    "\n",
    "mlp_optimizer = LBFGS(mlp_model.parameters())  \n",
    "\n",
    "for fold, (train_index, test_index) in enumerate(kf.split(X)):\n",
    "    # Prepare data for this fold\n",
    "    X_train, X_test = X[train_index], X[test_index]\n",
    "    y_train, y_test = y[train_index], y[test_index]\n",
    "\n",
    "    model = KAN(width=[X_train.shape[1], 2],  # Input features, 2 output classes\n",
    "            grid=5,          \n",
    "            k=3)             \n",
    "\n",
    "    # Ensure double-precision tensors\n",
    "    for param in model.parameters():\n",
    "        param.data = param.data.double()\n",
    "\n",
    "    optimizer = LBFGS(model.parameters()) \n",
    "\n",
    "\n",
    "\n",
    "    # --- MLP Model --- \n",
    "    mlp_model = Sequential(\n",
    "        Linear(X_train.shape[1], 8).double(),  # Input features \n",
    "        ReLU(),\n",
    "        Linear(8, 2).double()  # 2 output classes \n",
    "    )\n",
    "\n",
    "    # Ensure double-precision tensors\n",
    "    for param in mlp_model.parameters():\n",
    "        param.data = param.data.double()\n",
    "\n",
    "    mlp_optimizer = LBFGS(mlp_model.parameters())  \n",
    "\n",
    "# ... (Rest of the code: accuracy, training loop)\n",
    "    epochs = 20\n",
    "\n",
    "    for epoch in range(epochs):\n",
    "        ### KAN Training\n",
    "        def kan_closure():  \n",
    "            optimizer.zero_grad()\n",
    "            output = model(X_train)\n",
    "            loss =  CrossEntropyLoss()(output, y_train)\n",
    "            loss.backward()\n",
    "            return loss\n",
    "        optimizer.step(kan_closure)\n",
    "\n",
    "        ### MLP Training\n",
    "        def mlp_closure():\n",
    "            mlp_optimizer.zero_grad()\n",
    "            output = mlp_model(X_train)\n",
    "            loss = CrossEntropyLoss()(output, y_train)  # Using standard loss\n",
    "            loss.backward()\n",
    "            return loss\n",
    "        mlp_optimizer.step(mlp_closure)\n",
    "\n",
    "        kan_train_acc, kan_train_f1, kan_train_kappa = calculate_metrics(model(X_train), y_train)\n",
    "        kan_test_acc, kan_test_f1, kan_test_kappa = calculate_metrics(model(X_test), y_test) \n",
    "\n",
    "        mlp_train_acc, mlp_train_f1, mlp_train_kappa = calculate_metrics(mlp_model(X_train), y_train)\n",
    "        mlp_test_acc, mlp_test_f1, mlp_test_kappa = calculate_metrics(mlp_model(X_test), y_test)\n",
    "\n",
    "        print(f'KFOLD {fold}')\n",
    "        print(f'Epoch {epoch+1}/{epochs} | '\n",
    "            f'KAN Train Acc: {kan_train_acc:.3f} | KAN F1: {kan_train_f1:.3f} | KAN Kappa: {kan_train_kappa:.3f} | '\n",
    "            f'MLP Train Acc: {mlp_train_acc:.3f} | MLP F1: {mlp_train_f1:.3f} | MLP Kappa: {mlp_train_kappa:.3f}') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from mne.decoding import CSP\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA\n",
    "from sklearn.pipeline import make_pipeline\n",
    "import mne\n",
    "import moabb\n",
    "from moabb.datasets import BNCI2014_001\n",
    "from moabb.evaluations import WithinSessionEvaluation\n",
    "from moabb.paradigms import LeftRightImagery\n",
    "from mne.decoding import CSP\n",
    "\n",
    "\n",
    "moabb.set_log_level(\"info\")\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "csp = CSP(n_components=4, reg='oas') \n",
    "dataset = BNCI2014_001()\n",
    "dataset.subject_list = [1, 2, 3]\n",
    "sessions = dataset.get_data(subjects=[1])\n",
    "\n",
    "subject = 1\n",
    "session_name = \"0train\"\n",
    "run_name = \"0\"\n",
    "raw = sessions[subject][session_name][run_name]\n",
    "tmin, tmax = -1., 6.\n",
    "events = mne.find_events(raw, stim_channel=\"stim\")\n",
    "\n",
    "event_id = dict(left=1, right=2, feet=3, tongue=4)\n",
    "picks = pick_types(raw.info, meg=False, eeg=True, stim=False, eog=False, exclude='bads')\n",
    "epochs = Epochs(raw,\n",
    "                events,\n",
    "                event_id,\n",
    "                tmin,\n",
    "                tmax,\n",
    "                proj=True,\n",
    "                picks=picks,\n",
    "                baseline=None,\n",
    "                preload=True,\n",
    "                verbose=False)\n",
    "epochs.to_data_frame()\n",
    "#y = epochs.events[:, -1]\n",
    "\n",
    "#X = csp.fit_transform(1e6 *epochs.get_data(), y)  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "csp = CSP(n_components=4, reg='oas') \n",
    "data_e=data_EEG(multiclass=True, train=True)\n",
    "y = data_e.events[:, -1]\n",
    "X = csp.fit_transform( 1e6 *data_e.get_data(), y) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_EEG(multiclass=False, train=False).get_data().shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ... (Rest of the code: accuracy, training loop)\n",
    "epochs = 50\n",
    "kf = KFold(n_splits=10, shuffle=True, random_state=42)  # 5-fold cross-validation\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# Data Conversion (NumPy to PyTorch)\n",
    "\n",
    "\n",
    "model = KAN(width=[X.shape[1], 2],  # Input features, 2 output classes\n",
    "            grid=5,          \n",
    "            k=3)             \n",
    "\n",
    "# Ensure double-precision tensors\n",
    "for param in model.parameters():\n",
    "    param.data = param.data.double()\n",
    "\n",
    "optimizer = LBFGS(model.parameters()) \n",
    "\n",
    "\n",
    "\n",
    "# --- MLP Model --- \n",
    "mlp_model = Sequential(\n",
    "      Linear(X.shape[1], 8).double(),  # Input features \n",
    "      ReLU(),\n",
    "      Linear(8, 2).double()  # 2 output classes \n",
    ")\n",
    "\n",
    "# Ensure double-precision tensors\n",
    "for param in mlp_model.parameters():\n",
    "    param.data = param.data.double()\n",
    "\n",
    "mlp_optimizer = LBFGS(mlp_model.parameters())  \n",
    "\n",
    "for fold, (train_index, test_index) in enumerate(kf.split(X)):\n",
    "    # Prepare data for this fold\n",
    "    X_train, X_test = X[train_index], X[test_index]\n",
    "    y_train, y_test = y[train_index], y[test_index]\n",
    "\n",
    "    model = KAN(width=[X_train.shape[1], 2],  # Input features, 2 output classes\n",
    "            grid=5,          \n",
    "            k=3)             \n",
    "\n",
    "    # Ensure double-precision tensors\n",
    "    for param in model.parameters():\n",
    "        param.data = param.data.double()\n",
    "\n",
    "    optimizer = LBFGS(model.parameters()) \n",
    "\n",
    "\n",
    "\n",
    "    # --- MLP Model --- \n",
    "    mlp_model = Sequential(\n",
    "        Linear(X_train.shape[1], 8).double(),  # Input features \n",
    "        ReLU(),\n",
    "        Linear(8, 2).double()  # 2 output classes \n",
    "    )\n",
    "\n",
    "    # Ensure double-precision tensors\n",
    "    for param in mlp_model.parameters():\n",
    "        param.data = param.data.double()\n",
    "\n",
    "    mlp_optimizer = LBFGS(mlp_model.parameters())  \n",
    "\n",
    "# ... (Rest of the code: accuracy, training loop)\n",
    "    epochs = 20\n",
    "\n",
    "    for epoch in range(epochs):\n",
    "        ### KAN Training\n",
    "        def kan_closure():  \n",
    "            optimizer.zero_grad()\n",
    "            output = model(X_train)\n",
    "            loss =  CrossEntropyLoss()(output, y_train)\n",
    "            loss.backward()\n",
    "            return loss\n",
    "        optimizer.step(kan_closure)\n",
    "\n",
    "        ### MLP Training\n",
    "        def mlp_closure():\n",
    "            mlp_optimizer.zero_grad()\n",
    "            output = mlp_model(X_train)\n",
    "            loss = CrossEntropyLoss()(output, y_train)  # Using standard loss\n",
    "            loss.backward()\n",
    "            return loss\n",
    "        mlp_optimizer.step(mlp_closure)\n",
    "\n",
    "        kan_train_acc, kan_train_f1, kan_train_kappa = calculate_metrics(model(X_train), y_train)\n",
    "        kan_test_acc, kan_test_f1, kan_test_kappa = calculate_metrics(model(X_test), y_test) \n",
    "\n",
    "        mlp_train_acc, mlp_train_f1, mlp_train_kappa = calculate_metrics(mlp_model(X_train), y_train)\n",
    "        mlp_test_acc, mlp_test_f1, mlp_test_kappa = calculate_metrics(mlp_model(X_test), y_test)\n",
    "\n",
    "        print(f'KFOLD {fold}')\n",
    "        print(f'Epoch {epoch+1}/{epochs} | '\n",
    "            f'KAN Train Acc: {kan_train_acc:.3f} | KAN F1: {kan_train_f1:.3f} | KAN Kappa: {kan_train_kappa:.3f} | '\n",
    "            f'MLP Train Acc: {mlp_train_acc:.3f} | MLP F1: {mlp_train_f1:.3f} | MLP Kappa: {mlp_train_kappa:.3f}') "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}