KAN_MLP_BCI.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "from sklearn.model_selection import KFold\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from mne.decoding import CSP\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA\n",
    "from sklearn.pipeline import make_pipeline\n",
    "import mne\n",
    "import moabb\n",
    "from moabb.datasets import BNCI2014_001\n",
    "from moabb.evaluations import WithinSessionEvaluation\n",
    "from moabb.paradigms import LeftRightImagery\n",
    "from mne.decoding import CSP\n",
    "from mne import Epochs, pick_types, events_from_annotations\n",
    "from pykan.kan import KAN\n",
    "from sklearn.metrics import f1_score, cohen_kappa_score\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "import torch\n",
    "import numpy as np\n",
    "from torch.nn import Sequential, Linear, ReLU, CrossEntropyLoss, Module\n",
    "from pykan.kan import LBFGS \n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "#device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "device = torch.device('cpu')# if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "import torch.optim as optim\n",
    "from tqdm import tqdm\n",
    "import time\n",
    "import json\n",
    "\n",
    "\n",
    "epochs=20\n",
    "batch = 16\n",
    "lr = 0.1\n",
    "loss_fn = CrossEntropyLoss()\n",
    "steps=50\n",
    "kf = KFold(n_splits=10, shuffle=True, random_state=42)  \n",
    "subjects= [8]\n",
    "le = LabelEncoder()\n",
    "hidden_layers= 10\n",
    "\n",
    "moabb.set_log_level(\"info\")\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "csp = CSP(n_components=4, reg='oas') \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_EEG(subjects, train=True, multiclass=False):\n",
    "        if train:\n",
    "                session_name = \"0train\"\n",
    "\n",
    "        else:\n",
    "                session_name=\"1test\"\n",
    "\n",
    "        \n",
    "        dataset = BNCI2014_001()\n",
    "        if multiclass:\n",
    "                event_id = dict(left=1, right=2, feet=3, tongue=4)\n",
    "        else:\n",
    "                event_id = dict(left=1, right=2)#, feet=3, tongue=4)\n",
    "\n",
    "        tmin, tmax = -1., 6.\n",
    "        runs = range(5)\n",
    "\n",
    "        epochs_data=[]\n",
    "        for subject in subjects:\n",
    "                sessions = dataset.get_data(subjects=[subject])\n",
    "                for run in runs:\n",
    "                        raw = sessions[subject][session_name][str(run)]\n",
    "                        events = mne.find_events(raw, stim_channel=\"stim\")        \n",
    "                        picks = pick_types(raw.info, meg=False, eeg=True, stim=False, eog=False, exclude='bads')\n",
    "                        raw.filter(7., 35., method='iir', picks=picks)\n",
    "                        \n",
    "                        epochs = Epochs(raw,\n",
    "                                events,\n",
    "                                event_id,\n",
    "                                tmin,\n",
    "                                tmax,\n",
    "                                proj=True,\n",
    "                                picks=picks,\n",
    "                                baseline=None,\n",
    "                                preload=True,\n",
    "                                verbose=False)\n",
    "                        epochs_data.append(epochs)\n",
    "        return epochs_data\n",
    "\n",
    "def export_json(data, filename):\n",
    "    for key, value in data.items():\n",
    "        if isinstance(value, np.ndarray):\n",
    "            data[key] = value.tolist()\n",
    "        elif isinstance(value, list) and all(isinstance(x, list) for x in value):  # Check if ALL items are lists\n",
    "            data[key] = [[float(item) for item in sublist] for sublist in value]  # Convert inner elements to float\n",
    "        elif isinstance(value, list): \n",
    "            data[key] = [float(item) for item in value]  # Convert the numbers to float in the list \n",
    "\n",
    "\n",
    "    with open(filename, \"w\") as outfile:\n",
    "        json.dump(data, outfile, indent=4)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def creating_models(input_shape, output_shape):\n",
    "    kan_model = KAN(width=[input_shape, output_shape], grid=3, k=3, \n",
    "    seed=0, device=device)         \n",
    "\n",
    "    for param in kan_model.parameters():\n",
    "        param.data = param.data.double()\n",
    "        \n",
    "    return kan_model\n",
    "\n",
    "class MLP(Module):\n",
    "    def __init__(self, input_dim, hidden_dim, output_dim):\n",
    "        super().__init__()\n",
    "        self.layers = Sequential(\n",
    "            Linear(input_dim, hidden_dim),\n",
    "            ReLU(),  # Activation function (you can experiment with others)\n",
    "            Linear(hidden_dim, output_dim)  # Output layer\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.layers(x)\n",
    "\n",
    "\n",
    "    def train(self, dataset, opt=\"LBFGS\", steps=100, log=1, lamb=0., lamb_l1=1., lamb_entropy=2., lamb_coef=0., lamb_coefdiff=0., update_grid=True, grid_update_num=10, loss_fn=None, lr=1., stop_grid_update_step=50, batch=-1,\n",
    "              small_mag_threshold=1e-16, small_reg_factor=1., metrics=None, sglr_avoid=False, save_fig=False, in_vars=None, out_vars=None, beta=3, save_fig_freq=1, img_folder='./video', device='cpu'):\n",
    "        '''\n",
    "        training\n",
    "\n",
    "        Args:\n",
    "        -----\n",
    "            dataset : dic\n",
    "                contains dataset['train_input'], dataset['train_label'], dataset['test_input'], dataset['test_label']\n",
    "            opt : str\n",
    "                \"LBFGS\" or \"Adam\"\n",
    "            steps : int\n",
    "                training steps\n",
    "            log : int\n",
    "                logging frequency\n",
    "            lamb : float\n",
    "                overall penalty strength\n",
    "            lamb_l1 : float\n",
    "                l1 penalty strength\n",
    "            lamb_entropy : float\n",
    "                entropy penalty strength\n",
    "            lamb_coef : float\n",
    "                coefficient magnitude penalty strength\n",
    "            lamb_coefdiff : float\n",
    "                difference of nearby coefficits (smoothness) penalty strength\n",
    "            update_grid : bool\n",
    "                If True, update grid regularly before stop_grid_update_step\n",
    "            grid_update_num : int\n",
    "                the number of grid updates before stop_grid_update_step\n",
    "            stop_grid_update_step : int\n",
    "                no grid updates after this training step\n",
    "            batch : int\n",
    "                batch size, if -1 then full.\n",
    "            small_mag_threshold : float\n",
    "                threshold to determine large or small numbers (may want to apply larger penalty to smaller numbers)\n",
    "            small_reg_factor : float\n",
    "                penalty strength applied to small factors relative to large factos\n",
    "            device : str\n",
    "                device   \n",
    "            save_fig_freq : int\n",
    "                save figure every (save_fig_freq) step\n",
    "\n",
    "        Returns:\n",
    "        --------\n",
    "            results : dic\n",
    "                results['train_loss'], 1D array of training losses (RMSE)\n",
    "                results['test_loss'], 1D array of test losses (RMSE)\n",
    "                results['reg'], 1D array of regularization\n",
    "\n",
    "        Example\n",
    "        -------\n",
    "        >>> # for interactive examples, please see demos\n",
    "        >>> from utils import create_dataset\n",
    "        >>> model = KAN(width=[2,5,1], grid=5, k=3, noise_scale=0.1, seed=0)\n",
    "        >>> f = lambda x: torch.exp(torch.sin(torch.pi*x[:,[0]]) + x[:,[1]]**2)\n",
    "        >>> dataset = create_dataset(f, n_var=2)\n",
    "        >>> model.train(dataset, opt='LBFGS', steps=50, lamb=0.01);\n",
    "        >>> model.plot()\n",
    "        '''\n",
    "\n",
    "\n",
    "        pbar = tqdm(range(steps), desc='description', ncols=100)\n",
    "  \n",
    "        if loss_fn == None:\n",
    "            loss_fn = loss_fn_eval = lambda x, y: torch.mean((x - y) ** 2)\n",
    "        else:\n",
    "            loss_fn = loss_fn_eval = loss_fn\n",
    "\n",
    "        acc = acc_eval = lambda x,y: torch.mean((torch.argmax(x, dim=1) == y).float())\n",
    "\n",
    "        grid_update_freq = int(stop_grid_update_step / grid_update_num)\n",
    "\n",
    "        if opt == \"Adam\":\n",
    "            optimizer = torch.optim.Adam(self.parameters(), lr=lr)\n",
    "        elif opt == \"LBFGS\":\n",
    "            optimizer = LBFGS(self.parameters(), lr=lr, history_size=10, line_search_fn=\"strong_wolfe\", tolerance_grad=1e-32, tolerance_change=1e-32, tolerance_ys=1e-32)\n",
    "\n",
    "        results = {}\n",
    "        results['train_loss'] = []\n",
    "        results['test_loss'] = []\n",
    "        results['train_acc'] = []\n",
    "        results['test_acc'] = []\n",
    "        if metrics != None:\n",
    "            for i in range(len(metrics)):\n",
    "                results[metrics[i].__name__] = []\n",
    "\n",
    "        if batch == -1 or batch > dataset['train_input'].shape[0]:\n",
    "            batch_size = dataset['train_input'].shape[0]\n",
    "            batch_size_test = dataset['test_input'].shape[0]\n",
    "        else:\n",
    "            batch_size = batch\n",
    "            batch_size_test = batch\n",
    "\n",
    "        global train_loss, train_acc\n",
    "\n",
    "        def closure():\n",
    "            global train_loss, train_acc\n",
    "            optimizer.zero_grad()\n",
    "            pred = self.forward(dataset['train_input'][train_id].to(device))\n",
    "            if sglr_avoid == True:\n",
    "                id_ = torch.where(torch.isnan(torch.sum(pred, dim=1)) == False)[0]\n",
    "                train_loss = loss_fn(pred[id_], dataset['train_label'][train_id][id_].to(device))\n",
    "            else:\n",
    "                train_loss = loss_fn(pred, dataset['train_label'][train_id].to(device))\n",
    "            train_acc = acc(pred, dataset['train_label'][train_id].to(device))\n",
    "\n",
    "            objective = train_loss + lamb\n",
    "            objective.backward()\n",
    "            return objective\n",
    "\n",
    "        if save_fig:\n",
    "            if not os.path.exists(img_folder):\n",
    "                os.makedirs(img_folder)\n",
    "        \n",
    "        for _ in pbar:\n",
    "\n",
    "            train_id = np.random.choice(dataset['train_input'].shape[0], batch_size, replace=False)\n",
    "            test_id = np.random.choice(dataset['test_input'].shape[0], batch_size_test, replace=False)\n",
    "\n",
    "            #if _ % grid_update_freq == 0 and _ < stop_grid_update_step and update_grid:\n",
    "            #    self.update_grid_from_samples(dataset['train_input'][train_id].to(device))\n",
    "\n",
    "            if opt == \"LBFGS\":\n",
    "                optimizer.step(closure)\n",
    "\n",
    "            if opt == \"Adam\":\n",
    "                pred = self.forward(dataset['train_input'][train_id].to(device))\n",
    "                if sglr_avoid == True:\n",
    "                    id_ = torch.where(torch.isnan(torch.sum(pred, dim=1)) == False)[0]\n",
    "                    train_loss = loss_fn(pred[id_], dataset['train_label'][train_id][id_].to(device))\n",
    "                else:\n",
    "                    train_loss = loss_fn(pred, dataset['train_label'][train_id].to(device))\n",
    "                loss = train_loss + lamb\n",
    "                optimizer.zero_grad()\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "\n",
    "            test_loss = loss_fn_eval(self.forward(dataset['test_input'][test_id].to(device)), dataset['test_label'][test_id].to(device))\n",
    "            test_acc = acc_eval(self.forward(dataset['test_input'][test_id].to(device)), dataset['test_label'][test_id].to(device))\n",
    "\n",
    "            if _ % log == 0:\n",
    "                pbar.set_description(\"train acc: %.3f | test acc: %.3f \" % ((train_acc).cpu().detach().numpy(), (test_acc).cpu().detach().numpy()))\n",
    "\n",
    "            if metrics != None:\n",
    "                for i in range(len(metrics)):\n",
    "                    results[metrics[i].__name__].append(metrics[i]().item())\n",
    "\n",
    "            results['train_loss'].append(torch.sqrt(train_loss).cpu().detach().numpy())\n",
    "            results['test_loss'].append(torch.sqrt(test_loss).cpu().detach().numpy())\n",
    "            results['train_acc'].append(train_acc.cpu().detach().numpy())\n",
    "            results['test_acc'].append(test_acc.cpu().detach().numpy())\n",
    "\n",
    "            if save_fig and _ % save_fig_freq == 0:\n",
    "                self.plot(folder=img_folder, in_vars=in_vars, out_vars=out_vars, title=\"Step {}\".format(_), beta=beta)\n",
    "                plt.savefig(img_folder + '/' + str(_) + '.jpg', bbox_inches='tight', dpi=200)\n",
    "                plt.close()\n",
    "\n",
    "        return results\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1> TWO CLASS - RAW </h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.312 | reg: 14.121 : 100%|████████████| 50/50 [00:09<00:00,  5.32it/s]\n",
      "train acc: 0.875 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:00<00:00, 53.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.562 | reg: 22.644 : 100%|████████████| 50/50 [00:09<00:00,  5.32it/s]\n",
      "train acc: 0.812 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 53.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.312 | reg: 15.380 : 100%|████████████| 50/50 [00:09<00:00,  5.33it/s]\n",
      "train acc: 0.750 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 50.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.688 | reg: 14.540 : 100%|████████████| 50/50 [00:09<00:00,  5.37it/s]\n",
      "train acc: 0.750 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 53.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.625 | reg: 16.577 : 100%|████████████| 50/50 [00:09<00:00,  5.31it/s]\n",
      "train acc: 0.562 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 50.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.375 | reg: 15.390 : 100%|████████████| 50/50 [00:09<00:00,  5.36it/s]\n",
      "train acc: 0.812 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:01<00:00, 49.68it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.562 | reg: 15.958 : 100%|████████████| 50/50 [00:09<00:00,  5.37it/s]\n",
      "train acc: 0.812 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 52.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.375 | reg: 16.604 : 100%|████████████| 50/50 [00:09<00:00,  5.37it/s]\n",
      "train acc: 0.750 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:00<00:00, 52.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.500 | reg: 17.303 : 100%|████████████| 50/50 [00:09<00:00,  5.35it/s]\n",
      "train acc: 0.688 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:00<00:00, 54.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.375 | reg: 16.804 : 100%|████████████| 50/50 [00:09<00:00,  5.31it/s]\n",
      "train acc: 0.625 | test acc: 0.625 : 100%|██████████████████████████| 50/50 [00:00<00:00, 54.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.375 | reg: 15.723 : 100%|████████████| 50/50 [00:09<00:00,  5.27it/s]\n",
      "train acc: 0.625 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 53.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.500 | reg: 13.470 : 100%|████████████| 50/50 [00:09<00:00,  5.40it/s]\n",
      "train acc: 0.625 | test acc: 0.812 : 100%|██████████████████████████| 50/50 [00:00<00:00, 53.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.500 | reg: 14.081 : 100%|████████████| 50/50 [00:09<00:00,  5.29it/s]\n",
      "train acc: 0.562 | test acc: 0.688 : 100%|██████████████████████████| 50/50 [00:00<00:00, 51.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.438 | reg: 13.711 : 100%|████████████| 50/50 [00:09<00:00,  5.27it/s]\n",
      "train acc: 0.875 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 51.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.438 | reg: 20.063 : 100%|████████████| 50/50 [00:09<00:00,  5.31it/s]\n",
      "train acc: 0.812 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 52.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.312 | reg: 13.079 : 100%|████████████| 50/50 [00:09<00:00,  5.25it/s]\n",
      "train acc: 0.812 | test acc: 0.625 : 100%|██████████████████████████| 50/50 [00:00<00:00, 53.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.750 | reg: 18.106 : 100%|████████████| 50/50 [00:09<00:00,  5.36it/s]\n",
      "train acc: 0.625 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 50.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.562 | reg: 21.107 : 100%|████████████| 50/50 [00:09<00:00,  5.25it/s]\n",
      "train acc: 0.688 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 53.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.438 | reg: 16.656 : 100%|████████████| 50/50 [00:09<00:00,  5.28it/s]\n",
      "train acc: 0.750 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 52.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.625 | reg: 18.548 : 100%|████████████| 50/50 [00:09<00:00,  5.32it/s]\n",
      "train acc: 0.750 | test acc: 0.750 : 100%|██████████████████████████| 50/50 [00:01<00:00, 49.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.438 | reg: 19.990 : 100%|████████████| 50/50 [00:09<00:00,  5.35it/s]\n",
      "train acc: 0.750 | test acc: 0.625 : 100%|██████████████████████████| 50/50 [00:00<00:00, 53.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.250 | reg: 14.410 : 100%|████████████| 50/50 [00:09<00:00,  5.34it/s]\n",
      "train acc: 0.938 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:00<00:00, 53.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.688 | reg: 12.352 : 100%|████████████| 50/50 [00:09<00:00,  5.29it/s]\n",
      "train acc: 0.375 | test acc: 0.688 : 100%|██████████████████████████| 50/50 [00:00<00:00, 53.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.500 | reg: 13.646 : 100%|████████████| 50/50 [00:09<00:00,  5.26it/s]\n",
      "train acc: 0.750 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 50.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.562 | reg: 13.019 : 100%|████████████| 50/50 [00:09<00:00,  5.32it/s]\n",
      "train acc: 0.875 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 51.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.625 | reg: 13.661 : 100%|████████████| 50/50 [00:09<00:00,  5.27it/s]\n",
      "train acc: 0.812 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:01<00:00, 48.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.688 | reg: 13.909 : 100%|████████████| 50/50 [00:09<00:00,  5.32it/s]\n",
      "train acc: 0.875 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 54.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.438 | reg: 14.331 : 100%|████████████| 50/50 [00:09<00:00,  5.33it/s]\n",
      "train acc: 0.625 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 53.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.562 | reg: 19.050 : 100%|████████████| 50/50 [00:09<00:00,  5.29it/s]\n",
      "train acc: 0.562 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 54.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.625 | reg: 16.705 : 100%|████████████| 50/50 [00:12<00:00,  4.03it/s]\n",
      "train acc: 0.562 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:01<00:00, 35.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.312 | reg: 13.952 : 100%|████████████| 50/50 [00:14<00:00,  3.54it/s]\n",
      "train acc: 0.562 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 38.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.750 | reg: 15.914 : 100%|████████████| 50/50 [00:13<00:00,  3.79it/s]\n",
      "train acc: 0.812 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.500 | test acc: 0.500 | reg: 14.094 : 100%|████████████| 50/50 [00:13<00:00,  3.64it/s]\n",
      "train acc: 0.625 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:01<00:00, 39.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.500 | test acc: 0.500 | reg: 19.290 : 100%|████████████| 50/50 [00:16<00:00,  3.01it/s]\n",
      "train acc: 0.875 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:01<00:00, 34.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.438 | reg: 13.307 : 100%|████████████| 50/50 [00:14<00:00,  3.54it/s]\n",
      "train acc: 0.750 | test acc: 0.625 : 100%|██████████████████████████| 50/50 [00:01<00:00, 38.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.500 | reg: 14.232 : 100%|████████████| 50/50 [00:14<00:00,  3.56it/s]\n",
      "train acc: 0.688 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:01<00:00, 42.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.188 | reg: 14.168 : 100%|████████████| 50/50 [00:09<00:00,  5.32it/s]\n",
      "train acc: 0.625 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 52.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.625 | reg: 14.867 : 100%|████████████| 50/50 [00:09<00:00,  5.35it/s]\n",
      "train acc: 0.812 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.562 | reg: 16.230 : 100%|████████████| 50/50 [00:09<00:00,  5.36it/s]\n",
      "train acc: 0.625 | test acc: 0.750 : 100%|██████████████████████████| 50/50 [00:00<00:00, 52.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.562 | reg: 13.157 : 100%|████████████| 50/50 [00:09<00:00,  5.13it/s]\n",
      "train acc: 0.625 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 55.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.500 | reg: 15.093 : 100%|████████████| 50/50 [00:09<00:00,  5.40it/s]\n",
      "train acc: 0.688 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 58.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.500 | reg: 13.867 : 100%|████████████| 50/50 [00:09<00:00,  5.34it/s]\n",
      "train acc: 0.562 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 52.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.625 | reg: 18.673 : 100%|████████████| 50/50 [00:09<00:00,  5.43it/s]\n",
      "train acc: 0.562 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.750 | reg: 13.796 : 100%|████████████| 50/50 [00:09<00:00,  5.42it/s]\n",
      "train acc: 0.562 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.562 | reg: 18.152 : 100%|████████████| 50/50 [00:09<00:00,  5.40it/s]\n",
      "train acc: 0.562 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 55.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.688 | reg: 25.888 : 100%|████████████| 50/50 [00:09<00:00,  5.42it/s]\n",
      "train acc: 0.500 | test acc: 0.688 : 100%|██████████████████████████| 50/50 [00:00<00:00, 56.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.188 | reg: 14.985 : 100%|████████████| 50/50 [00:09<00:00,  5.35it/s]\n",
      "train acc: 0.750 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:00<00:00, 53.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.375 | reg: 15.818 : 100%|████████████| 50/50 [00:09<00:00,  5.32it/s]\n",
      "train acc: 0.812 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 51.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.500 | reg: 15.297 : 100%|████████████| 50/50 [00:09<00:00,  5.39it/s]\n",
      "train acc: 0.875 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 55.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.500 | reg: 16.306 : 100%|████████████| 50/50 [00:09<00:00,  5.50it/s]\n",
      "train acc: 0.812 | test acc: 0.625 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.438 | reg: 17.252 : 100%|████████████| 50/50 [00:09<00:00,  5.51it/s]\n",
      "train acc: 0.938 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 55.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.500 | reg: 14.777 : 100%|████████████| 50/50 [00:09<00:00,  5.51it/s]\n",
      "train acc: 0.750 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.562 | reg: 18.759 : 100%|████████████| 50/50 [00:09<00:00,  5.25it/s]\n",
      "train acc: 0.750 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:00<00:00, 52.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.438 | reg: 15.824 : 100%|████████████| 50/50 [00:09<00:00,  5.35it/s]\n",
      "train acc: 0.688 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 56.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.625 | reg: 16.114 : 100%|████████████| 50/50 [00:09<00:00,  5.37it/s]\n",
      "train acc: 0.812 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.562 | reg: 14.602 : 100%|████████████| 50/50 [00:09<00:00,  5.40it/s]\n",
      "train acc: 0.812 | test acc: 0.750 : 100%|██████████████████████████| 50/50 [00:00<00:00, 56.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.625 | reg: 15.291 : 100%|████████████| 50/50 [00:09<00:00,  5.45it/s]\n",
      "train acc: 0.812 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:00<00:00, 54.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.750 | reg: 13.303 : 100%|████████████| 50/50 [00:09<00:00,  5.39it/s]\n",
      "train acc: 0.875 | test acc: 0.688 : 100%|██████████████████████████| 50/50 [00:00<00:00, 58.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.688 | reg: 14.092 : 100%|████████████| 50/50 [00:09<00:00,  5.49it/s]\n",
      "train acc: 0.750 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.562 | reg: 16.859 : 100%|████████████| 50/50 [00:09<00:00,  5.46it/s]\n",
      "train acc: 0.750 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.438 | reg: 16.446 : 100%|████████████| 50/50 [00:09<00:00,  5.35it/s]\n",
      "train acc: 0.688 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 53.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.562 | reg: 11.569 : 100%|████████████| 50/50 [00:09<00:00,  5.28it/s]\n",
      "train acc: 0.625 | test acc: 0.750 : 100%|██████████████████████████| 50/50 [00:00<00:00, 56.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.500 | reg: 15.387 : 100%|████████████| 50/50 [00:09<00:00,  5.35it/s]\n",
      "train acc: 0.875 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:00<00:00, 55.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.438 | reg: 19.687 : 100%|████████████| 50/50 [00:09<00:00,  5.38it/s]\n",
      "train acc: 0.625 | test acc: 0.750 : 100%|██████████████████████████| 50/50 [00:00<00:00, 55.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.562 | reg: 18.370 : 100%|████████████| 50/50 [00:09<00:00,  5.32it/s]\n",
      "train acc: 0.812 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.562 | reg: 17.241 : 100%|████████████| 50/50 [00:09<00:00,  5.29it/s]\n",
      "train acc: 0.938 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:00<00:00, 51.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.562 | reg: 17.365 : 100%|████████████| 50/50 [00:10<00:00,  4.98it/s]\n",
      "train acc: 0.875 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:01<00:00, 42.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.500 | reg: 16.291 : 100%|████████████| 50/50 [00:12<00:00,  3.85it/s]\n",
      "train acc: 0.812 | test acc: 0.750 : 100%|██████████████████████████| 50/50 [00:01<00:00, 39.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.562 | reg: 15.396 : 100%|████████████| 50/50 [00:10<00:00,  4.94it/s]\n",
      "train acc: 0.625 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 58.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.625 | reg: 13.495 : 100%|████████████| 50/50 [00:09<00:00,  5.50it/s]\n",
      "train acc: 0.562 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.500 | reg: 13.998 : 100%|████████████| 50/50 [00:09<00:00,  5.47it/s]\n",
      "train acc: 0.562 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 56.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.562 | reg: 12.172 : 100%|████████████| 50/50 [00:09<00:00,  5.53it/s]\n",
      "train acc: 0.688 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.438 | reg: 15.463 : 100%|████████████| 50/50 [00:08<00:00,  5.58it/s]\n",
      "train acc: 0.750 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 54.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.438 | reg: 11.914 : 100%|████████████| 50/50 [00:08<00:00,  5.63it/s]\n",
      "train acc: 0.562 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.625 | reg: 15.524 : 100%|████████████| 50/50 [00:08<00:00,  5.59it/s]\n",
      "train acc: 0.750 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 54.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.750 | reg: 14.993 : 100%|████████████| 50/50 [00:09<00:00,  5.49it/s]\n",
      "train acc: 0.438 | test acc: 0.750 : 100%|██████████████████████████| 50/50 [00:00<00:00, 54.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.625 | reg: 14.127 : 100%|████████████| 50/50 [00:08<00:00,  5.56it/s]\n",
      "train acc: 0.625 | test acc: 0.750 : 100%|██████████████████████████| 50/50 [00:00<00:00, 55.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.688 | reg: 12.579 : 100%|████████████| 50/50 [00:09<00:00,  5.48it/s]\n",
      "train acc: 0.812 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.375 | reg: 12.503 : 100%|████████████| 50/50 [00:09<00:00,  5.42it/s]\n",
      "train acc: 0.625 | test acc: 0.625 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.500 | reg: 13.212 : 100%|████████████| 50/50 [00:09<00:00,  5.43it/s]\n",
      "train acc: 0.875 | test acc: 0.750 : 100%|██████████████████████████| 50/50 [00:00<00:00, 55.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.562 | reg: 14.185 : 100%|████████████| 50/50 [00:09<00:00,  5.48it/s]\n",
      "train acc: 0.875 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 56.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.625 | reg: 13.801 : 100%|████████████| 50/50 [00:09<00:00,  5.40it/s]\n",
      "train acc: 0.562 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 56.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.625 | reg: 13.360 : 100%|████████████| 50/50 [00:09<00:00,  5.38it/s]\n",
      "train acc: 0.812 | test acc: 0.688 : 100%|██████████████████████████| 50/50 [00:00<00:00, 55.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.375 | reg: 13.441 : 100%|████████████| 50/50 [00:09<00:00,  5.38it/s]\n",
      "train acc: 0.625 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.562 | reg: 12.714 : 100%|████████████| 50/50 [00:09<00:00,  5.55it/s]\n",
      "train acc: 0.688 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 58.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.688 | reg: 15.234 : 100%|████████████| 50/50 [00:09<00:00,  5.53it/s]\n",
      "train acc: 0.812 | test acc: 0.625 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.438 | reg: 12.997 : 100%|████████████| 50/50 [00:09<00:00,  5.49it/s]\n",
      "train acc: 0.750 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 55.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.250 | reg: 10.766 : 100%|████████████| 50/50 [00:09<00:00,  5.41it/s]\n",
      "train acc: 0.562 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:00<00:00, 54.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.688 | reg: 12.210 : 100%|████████████| 50/50 [00:09<00:00,  5.45it/s]\n",
      "train acc: 0.812 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 56.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.312 | reg: 13.387 : 100%|████████████| 50/50 [00:09<00:00,  5.44it/s]\n",
      "train acc: 0.750 | test acc: 0.688 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.562 | reg: 13.025 : 100%|████████████| 50/50 [00:09<00:00,  5.40it/s]\n",
      "train acc: 0.750 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.688 | reg: 13.135 : 100%|████████████| 50/50 [00:09<00:00,  5.44it/s]\n",
      "train acc: 0.625 | test acc: 0.625 : 100%|██████████████████████████| 50/50 [00:00<00:00, 52.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.500 | reg: 11.766 : 100%|████████████| 50/50 [00:10<00:00,  4.75it/s]\n",
      "train acc: 0.812 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:01<00:00, 37.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.625 | reg: 12.319 : 100%|████████████| 50/50 [00:10<00:00,  4.68it/s]\n",
      "train acc: 0.312 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.562 | reg: 12.999 : 100%|████████████| 50/50 [00:09<00:00,  5.42it/s]\n",
      "train acc: 0.562 | test acc: 0.688 : 100%|██████████████████████████| 50/50 [00:00<00:00, 58.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.688 | reg: 12.911 : 100%|████████████| 50/50 [00:09<00:00,  5.50it/s]\n",
      "train acc: 0.500 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 56.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.500 | reg: 12.213 : 100%|████████████| 50/50 [00:09<00:00,  5.41it/s]\n",
      "train acc: 0.750 | test acc: 0.750 : 100%|██████████████████████████| 50/50 [00:00<00:00, 58.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.625 | reg: 11.849 : 100%|████████████| 50/50 [00:11<00:00,  4.22it/s]\n",
      "train acc: 0.688 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:01<00:00, 43.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.438 | reg: 13.022 : 100%|████████████| 50/50 [00:13<00:00,  3.78it/s]\n",
      "train acc: 0.500 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:01<00:00, 38.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.688 | reg: 13.293 : 100%|████████████| 50/50 [00:12<00:00,  3.90it/s]\n",
      "train acc: 0.812 | test acc: 0.688 : 100%|██████████████████████████| 50/50 [00:01<00:00, 43.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.625 | reg: 13.042 : 100%|████████████| 50/50 [00:13<00:00,  3.70it/s]\n",
      "train acc: 0.625 | test acc: 0.750 : 100%|██████████████████████████| 50/50 [00:01<00:00, 39.68it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.500 | reg: 13.453 : 100%|████████████| 50/50 [00:12<00:00,  3.92it/s]\n",
      "train acc: 0.562 | test acc: 0.688 : 100%|██████████████████████████| 50/50 [00:01<00:00, 36.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.438 | reg: 13.010 : 100%|████████████| 50/50 [00:12<00:00,  3.99it/s]\n",
      "train acc: 0.562 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:01<00:00, 42.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.688 | reg: 12.036 : 100%|████████████| 50/50 [00:12<00:00,  3.91it/s]\n",
      "train acc: 0.812 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 42.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.312 | reg: 12.599 : 100%|████████████| 50/50 [00:12<00:00,  4.08it/s]\n",
      "train acc: 0.750 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 52.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.562 | reg: 13.236 : 100%|████████████| 50/50 [00:09<00:00,  5.50it/s]\n",
      "train acc: 0.688 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.812 | reg: 12.972 : 100%|████████████| 50/50 [00:08<00:00,  5.62it/s]\n",
      "train acc: 0.562 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 53.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.562 | reg: 14.933 : 100%|████████████| 50/50 [00:09<00:00,  5.46it/s]\n",
      "train acc: 0.875 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:00<00:00, 58.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.625 | reg: 12.928 : 100%|████████████| 50/50 [00:08<00:00,  5.62it/s]\n",
      "train acc: 0.500 | test acc: 0.625 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.250 | reg: 13.542 : 100%|████████████| 50/50 [00:09<00:00,  5.41it/s]\n",
      "train acc: 0.625 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 58.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.312 | reg: 13.740 : 100%|████████████| 50/50 [00:09<00:00,  5.47it/s]\n",
      "train acc: 0.875 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 55.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.500 | reg: 13.703 : 100%|████████████| 50/50 [00:09<00:00,  5.46it/s]\n",
      "train acc: 0.438 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 54.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.438 | reg: 13.161 : 100%|████████████| 50/50 [00:09<00:00,  5.52it/s]\n",
      "train acc: 0.625 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 55.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.562 | reg: 13.596 : 100%|████████████| 50/50 [00:09<00:00,  5.46it/s]\n",
      "train acc: 0.812 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:00<00:00, 52.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.562 | reg: 13.000 : 100%|████████████| 50/50 [00:09<00:00,  5.39it/s]\n",
      "train acc: 1.000 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 56.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.625 | reg: 13.178 : 100%|████████████| 50/50 [00:09<00:00,  5.37it/s]\n",
      "train acc: 0.438 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 56.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.562 | reg: 14.360 : 100%|████████████| 50/50 [00:09<00:00,  5.51it/s]\n",
      "train acc: 0.875 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 58.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.375 | reg: 15.453 : 100%|████████████| 50/50 [00:08<00:00,  5.57it/s]\n",
      "train acc: 0.562 | test acc: 0.625 : 100%|██████████████████████████| 50/50 [00:00<00:00, 58.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.375 | reg: 14.787 : 100%|████████████| 50/50 [00:09<00:00,  5.53it/s]\n",
      "train acc: 0.562 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.562 | reg: 13.454 : 100%|████████████| 50/50 [00:09<00:00,  5.47it/s]\n",
      "train acc: 0.750 | test acc: 0.688 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.78it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.438 | reg: 13.347 : 100%|████████████| 50/50 [00:09<00:00,  5.41it/s]\n",
      "train acc: 0.875 | test acc: 0.625 : 100%|██████████████████████████| 50/50 [00:00<00:00, 56.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.625 | reg: 14.273 : 100%|████████████| 50/50 [00:09<00:00,  5.37it/s]\n",
      "train acc: 0.750 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.375 | reg: 17.538 : 100%|████████████| 50/50 [00:09<00:00,  5.46it/s]\n",
      "train acc: 0.625 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.500 | reg: 14.064 : 100%|████████████| 50/50 [00:09<00:00,  5.38it/s]\n",
      "train acc: 0.812 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.375 | reg: 19.312 : 100%|████████████| 50/50 [00:09<00:00,  5.41it/s]\n",
      "train acc: 0.812 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.562 | reg: 15.697 : 100%|████████████| 50/50 [00:09<00:00,  5.46it/s]\n",
      "train acc: 0.688 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.500 | reg: 12.395 : 100%|████████████| 50/50 [00:09<00:00,  5.47it/s]\n",
      "train acc: 0.750 | test acc: 0.625 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.438 | reg: 13.498 : 100%|████████████| 50/50 [00:09<00:00,  5.47it/s]\n",
      "train acc: 0.562 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 54.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.188 | reg: 12.132 : 100%|████████████| 50/50 [00:08<00:00,  5.58it/s]\n",
      "train acc: 0.688 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 58.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.688 | reg: 12.649 : 100%|████████████| 50/50 [00:09<00:00,  5.56it/s]\n",
      "train acc: 0.562 | test acc: 0.750 : 100%|██████████████████████████| 50/50 [00:00<00:00, 58.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.438 | reg: 11.054 : 100%|████████████| 50/50 [00:09<00:00,  5.52it/s]\n",
      "train acc: 0.562 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.375 | reg: 12.024 : 100%|████████████| 50/50 [00:09<00:00,  5.49it/s]\n",
      "train acc: 0.750 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:00<00:00, 56.71it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.500 | reg: 13.476 : 100%|████████████| 50/50 [00:09<00:00,  5.45it/s]\n",
      "train acc: 0.750 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 55.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.625 | reg: 16.664 : 100%|████████████| 50/50 [00:13<00:00,  3.84it/s]\n",
      "train acc: 0.688 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 40.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.500 | reg: 11.271 : 100%|████████████| 50/50 [00:12<00:00,  3.97it/s]\n",
      "train acc: 0.688 | test acc: 0.688 : 100%|██████████████████████████| 50/50 [00:01<00:00, 43.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.562 | reg: 13.952 : 100%|████████████| 50/50 [00:13<00:00,  3.73it/s]\n",
      "train acc: 0.812 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:01<00:00, 41.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.688 | reg: 15.255 : 100%|████████████| 50/50 [00:12<00:00,  3.92it/s]\n",
      "train acc: 0.500 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:01<00:00, 37.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.500 | reg: 16.151 : 100%|████████████| 50/50 [00:12<00:00,  3.89it/s]\n",
      "train acc: 0.688 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.625 | reg: 14.589 : 100%|████████████| 50/50 [00:12<00:00,  3.90it/s]\n",
      "train acc: 0.938 | test acc: 0.688 : 100%|██████████████████████████| 50/50 [00:01<00:00, 34.48it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.625 | reg: 13.938 : 100%|████████████| 50/50 [00:12<00:00,  4.06it/s]\n",
      "train acc: 0.562 | test acc: 0.625 : 100%|██████████████████████████| 50/50 [00:01<00:00, 40.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.438 | reg: 18.037 : 100%|████████████| 50/50 [00:12<00:00,  3.96it/s]\n",
      "train acc: 0.875 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:01<00:00, 35.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.562 | reg: 13.440 : 100%|████████████| 50/50 [00:12<00:00,  3.98it/s]\n",
      "train acc: 0.812 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 34.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.438 | reg: 20.216 : 100%|████████████| 50/50 [00:13<00:00,  3.74it/s]\n",
      "train acc: 0.812 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:01<00:00, 41.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.625 | reg: 15.795 : 100%|████████████| 50/50 [00:12<00:00,  3.95it/s]\n",
      "train acc: 0.625 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 50.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.500 | reg: 17.060 : 100%|████████████| 50/50 [00:12<00:00,  3.92it/s]\n",
      "train acc: 0.375 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:01<00:00, 30.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.500 | reg: 15.578 : 100%|████████████| 50/50 [00:13<00:00,  3.82it/s]\n",
      "train acc: 0.875 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:01<00:00, 32.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.438 | reg: 16.171 : 100%|████████████| 50/50 [00:13<00:00,  3.75it/s]\n",
      "train acc: 0.688 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:01<00:00, 34.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.562 | reg: 15.747 : 100%|████████████| 50/50 [00:13<00:00,  3.59it/s]\n",
      "train acc: 0.625 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:01<00:00, 33.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.562 | reg: 13.577 : 100%|████████████| 50/50 [00:13<00:00,  3.80it/s]\n",
      "train acc: 0.750 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:01<00:00, 39.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.562 | reg: 17.962 : 100%|████████████| 50/50 [00:12<00:00,  3.89it/s]\n",
      "train acc: 0.625 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:01<00:00, 42.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.500 | reg: 19.830 : 100%|████████████| 50/50 [00:13<00:00,  3.69it/s]\n",
      "train acc: 0.688 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:01<00:00, 34.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.562 | reg: 16.552 : 100%|████████████| 50/50 [00:13<00:00,  3.73it/s]\n",
      "train acc: 0.688 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:01<00:00, 37.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.375 | reg: 12.365 : 100%|████████████| 50/50 [00:12<00:00,  4.04it/s]\n",
      "train acc: 0.625 | test acc: 0.688 : 100%|██████████████████████████| 50/50 [00:00<00:00, 55.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.250 | reg: 14.362 : 100%|████████████| 50/50 [00:09<00:00,  5.34it/s]\n",
      "train acc: 0.938 | test acc: 0.750 : 100%|██████████████████████████| 50/50 [00:00<00:00, 53.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.625 | reg: 17.373 : 100%|████████████| 50/50 [00:09<00:00,  5.34it/s]\n",
      "train acc: 0.750 | test acc: 0.625 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.438 | reg: 14.170 : 100%|████████████| 50/50 [00:09<00:00,  5.30it/s]\n",
      "train acc: 0.750 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 55.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.625 | reg: 17.097 : 100%|████████████| 50/50 [00:09<00:00,  5.37it/s]\n",
      "train acc: 0.625 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 55.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.500 | reg: 14.837 : 100%|████████████| 50/50 [00:09<00:00,  5.31it/s]\n",
      "train acc: 0.688 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 52.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.312 | reg: 15.787 : 100%|████████████| 50/50 [00:09<00:00,  5.26it/s]\n",
      "train acc: 0.562 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 55.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.562 | reg: 13.993 : 100%|████████████| 50/50 [00:09<00:00,  5.34it/s]\n",
      "train acc: 0.812 | test acc: 0.625 : 100%|██████████████████████████| 50/50 [00:00<00:00, 53.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.562 | reg: 13.880 : 100%|████████████| 50/50 [00:09<00:00,  5.26it/s]\n",
      "train acc: 0.875 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 54.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.375 | reg: 15.211 : 100%|████████████| 50/50 [00:09<00:00,  5.32it/s]\n",
      "train acc: 0.750 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 53.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.750 | reg: 13.718 : 100%|████████████| 50/50 [00:09<00:00,  5.34it/s]\n",
      "train acc: 0.688 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 52.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.438 | reg: 11.086 : 100%|████████████| 50/50 [00:09<00:00,  5.35it/s]\n",
      "train acc: 0.688 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:01<00:00, 49.71it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.375 | reg: 12.987 : 100%|████████████| 50/50 [00:09<00:00,  5.27it/s]\n",
      "train acc: 0.750 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 53.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.375 | reg: 13.291 : 100%|████████████| 50/50 [00:09<00:00,  5.31it/s]\n",
      "train acc: 0.875 | test acc: 0.625 : 100%|██████████████████████████| 50/50 [00:00<00:00, 52.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.688 | reg: 13.725 : 100%|████████████| 50/50 [00:09<00:00,  5.32it/s]\n",
      "train acc: 0.812 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:00<00:00, 53.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.250 | reg: 12.833 : 100%|████████████| 50/50 [00:09<00:00,  5.39it/s]\n",
      "train acc: 0.500 | test acc: 0.812 : 100%|██████████████████████████| 50/50 [00:00<00:00, 54.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.500 | reg: 14.184 : 100%|████████████| 50/50 [00:09<00:00,  5.39it/s]\n",
      "train acc: 0.750 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 53.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.312 | reg: 16.325 : 100%|████████████| 50/50 [00:09<00:00,  5.42it/s]\n",
      "train acc: 0.938 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:00<00:00, 54.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.625 | reg: 13.045 : 100%|████████████| 50/50 [00:09<00:00,  5.25it/s]\n",
      "train acc: 1.000 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 54.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.625 | reg: 12.199 : 100%|████████████| 50/50 [00:09<00:00,  5.30it/s]\n",
      "train acc: 0.875 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 51.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.438 | reg: 15.365 : 100%|████████████| 50/50 [00:09<00:00,  5.22it/s]\n",
      "train acc: 0.750 | test acc: 0.625 : 100%|██████████████████████████| 50/50 [00:00<00:00, 51.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.500 | reg: 14.497 : 100%|████████████| 50/50 [00:09<00:00,  5.31it/s]\n",
      "train acc: 0.562 | test acc: 0.125 : 100%|██████████████████████████| 50/50 [00:00<00:00, 50.78it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.375 | reg: 13.036 : 100%|████████████| 50/50 [00:09<00:00,  5.35it/s]\n",
      "train acc: 0.812 | test acc: 0.625 : 100%|██████████████████████████| 50/50 [00:00<00:00, 54.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.500 | reg: 15.326 : 100%|████████████| 50/50 [00:09<00:00,  5.31it/s]\n",
      "train acc: 0.438 | test acc: 0.812 : 100%|██████████████████████████| 50/50 [00:00<00:00, 51.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.375 | reg: 15.049 : 100%|████████████| 50/50 [00:09<00:00,  5.28it/s]\n",
      "train acc: 0.938 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:00<00:00, 53.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.562 | reg: 12.543 : 100%|████████████| 50/50 [00:09<00:00,  5.30it/s]\n",
      "train acc: 0.938 | test acc: 0.688 : 100%|██████████████████████████| 50/50 [00:00<00:00, 51.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.375 | reg: 13.597 : 100%|████████████| 50/50 [00:09<00:00,  5.26it/s]\n",
      "train acc: 0.562 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:00<00:00, 53.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.312 | reg: 12.955 : 100%|████████████| 50/50 [00:09<00:00,  5.30it/s]\n",
      "train acc: 0.875 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 49.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.500 | test acc: 0.562 | reg: 13.054 : 100%|████████████| 50/50 [00:10<00:00,  4.58it/s]\n",
      "train acc: 0.812 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:01<00:00, 37.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.312 | reg: 13.758 : 100%|████████████| 50/50 [00:13<00:00,  3.81it/s]\n",
      "train acc: 0.688 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:01<00:00, 38.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.438 | reg: 13.234 : 100%|████████████| 50/50 [00:10<00:00,  4.89it/s]\n",
      "train acc: 0.500 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 54.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.438 | reg: 17.551 : 100%|████████████| 50/50 [00:09<00:00,  5.33it/s]\n",
      "train acc: 0.812 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 50.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.688 | reg: 15.292 : 100%|████████████| 50/50 [00:09<00:00,  5.31it/s]\n",
      "train acc: 0.688 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 53.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.562 | reg: 17.380 : 100%|████████████| 50/50 [00:09<00:00,  5.29it/s]\n",
      "train acc: 0.750 | test acc: 0.625 : 100%|██████████████████████████| 50/50 [00:01<00:00, 49.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.312 | reg: 12.956 : 100%|████████████| 50/50 [00:09<00:00,  5.31it/s]\n",
      "train acc: 0.562 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 53.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.438 | reg: 14.031 : 100%|████████████| 50/50 [00:09<00:00,  5.33it/s]\n",
      "train acc: 0.750 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:01<00:00, 49.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.625 | reg: 11.127 : 100%|████████████| 50/50 [00:09<00:00,  5.32it/s]\n",
      "train acc: 0.750 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 54.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.438 | reg: 11.923 : 100%|████████████| 50/50 [00:09<00:00,  5.37it/s]\n",
      "train acc: 0.625 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:00<00:00, 52.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.375 | reg: 14.237 : 100%|████████████| 50/50 [00:09<00:00,  5.39it/s]\n",
      "train acc: 0.500 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 54.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.250 | reg: 13.418 : 100%|████████████| 50/50 [00:09<00:00,  5.30it/s]\n",
      "train acc: 0.500 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:00<00:00, 51.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.750 | reg: 12.322 : 100%|████████████| 50/50 [00:09<00:00,  5.32it/s]\n",
      "train acc: 0.812 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:00<00:00, 53.78it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.688 | reg: 12.656 : 100%|████████████| 50/50 [00:09<00:00,  5.33it/s]\n",
      "train acc: 0.812 | test acc: 0.875 : 100%|██████████████████████████| 50/50 [00:00<00:00, 51.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.250 | reg: 13.446 : 100%|████████████| 50/50 [00:09<00:00,  5.35it/s]\n",
      "train acc: 0.812 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:00<00:00, 53.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.500 | reg: 12.289 : 100%|████████████| 50/50 [00:09<00:00,  5.33it/s]\n",
      "train acc: 0.812 | test acc: 0.562 : 100%|██████████████████████████| 50/50 [00:00<00:00, 52.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.688 | reg: 13.616 : 100%|████████████| 50/50 [00:09<00:00,  5.33it/s]\n",
      "train acc: 0.562 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:00<00:00, 54.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.688 | reg: 15.874 : 100%|████████████| 50/50 [00:10<00:00,  4.90it/s]\n",
      "train acc: 0.688 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:01<00:00, 37.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.625 | reg: 11.413 : 100%|████████████| 50/50 [00:14<00:00,  3.41it/s]\n",
      "train acc: 0.625 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 41.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.750 | reg: 13.862 : 100%|████████████| 50/50 [00:13<00:00,  3.70it/s]\n",
      "train acc: 0.750 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 41.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training time KAN: 2016.92 seconds\n",
      "Training time MLP: 198.72 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "epochs_EEG=mne.concatenate_epochs(data_EEG(subjects=subjects))\n",
    "df = epochs_EEG.to_data_frame().drop(columns=['time','epoch'])\n",
    "X = df.drop(columns=['condition']).values\n",
    "y = df.condition.values\n",
    "y = le.fit_transform(y)\n",
    "\n",
    "X = torch.from_numpy(X).double()\n",
    "y  = torch.from_numpy(y).long()\n",
    "\n",
    "\n",
    "input_layer = X.shape[1]\n",
    "output_layer = len(np.unique(y))\n",
    "\n",
    "\n",
    "kan_model  = creating_models(input_layer, output_layer)\n",
    "kan_optimizer = LBFGS(kan_model.parameters(), lr=lr) \n",
    "\n",
    "mlp_model = MLP(input_layer, hidden_layers, output_layer).to(device)\n",
    "\n",
    "for param in mlp_model.parameters():\n",
    "    param.data = param.data.double()\n",
    "mlp_optimizer = LBFGS(mlp_model.parameters(), lr=lr, history_size=10, line_search_fn=\"strong_wolfe\", tolerance_grad=1e-32, tolerance_change=1e-32, tolerance_ys=1e-32) #(mlp_model.parameters(), lr=lr)\n",
    "\n",
    "\n",
    "metrics_KAN = {}  \n",
    "metrics_KAN['train'] = []\n",
    "metrics_KAN['test'] = []\n",
    "\n",
    "metrics_MLP = {}  \n",
    "metrics_MLP['train'] = []\n",
    "metrics_MLP['test'] = []\n",
    "\n",
    "total_train_time_KAN = 0\n",
    "total_train_time_MLP = 0\n",
    "\n",
    "for fold, (train_index, test_index) in enumerate(kf.split(X)):\n",
    "\n",
    "    X_train, X_test = X[train_index], X[test_index]\n",
    "    y_train, y_test = y[train_index], y[test_index]\n",
    "\n",
    "    dataset={}\n",
    "    dataset['train_input']= X_train\n",
    "    dataset['train_label']= y_train\n",
    "    dataset['test_input']= X_test\n",
    "    dataset['test_label']= y_test\n",
    "\n",
    "    fold_metrics_KAN_train = []\n",
    "    fold_metrics_KAN_test =[]\n",
    "    fold_metrics_MLP_train = []\n",
    "    fold_metrics_MLP_test =[]\n",
    "\n",
    "    for epoch in range(epochs):\n",
    "        print('###### EPOCH ', epoch, \" ############\")\n",
    "\n",
    "        epoch_start_time_KAN = time.time()\n",
    "\n",
    "        results_kan = kan_model.train(dataset, opt=\"LBFGS\", steps=steps, \n",
    "        loss_fn=loss_fn, batch=batch, lr=lr)\n",
    "        \n",
    "        epoch_end_time_KAN = time.time()\n",
    "        epoch_train_KAN = epoch_end_time_KAN - epoch_start_time_KAN\n",
    "        total_train_time_KAN += epoch_train_KAN\n",
    "        \n",
    "        epoch_start_time_MLP = time.time()\n",
    "\n",
    "        fold_metrics_KAN_train.append(np.mean(results_kan['train_acc']))\n",
    "        fold_metrics_KAN_test.append(np.mean(results_kan['test_acc'])) \n",
    "\n",
    "        results_mlp = mlp_model.train(dataset, opt=\"LBFGS\", steps=steps, \n",
    "        loss_fn=loss_fn, batch=batch, lr=lr)\n",
    "\n",
    "        epoch_end_time_MLP = time.time()\n",
    "        epoch_train_MLP = epoch_end_time_MLP - epoch_start_time_MLP\n",
    "        total_train_time_MLP += epoch_train_MLP\n",
    "        \n",
    "        fold_metrics_MLP_train.append(np.mean(results_mlp['train_acc']))\n",
    "        fold_metrics_MLP_test.append(np.mean(results_mlp['test_acc'])) \n",
    "\n",
    "    metrics_KAN['train'].append(fold_metrics_KAN_train)\n",
    "    metrics_KAN['test'].append(fold_metrics_KAN_test)\n",
    "    \n",
    "    metrics_MLP['train'].append(fold_metrics_MLP_train)\n",
    "    metrics_MLP['test'].append(fold_metrics_MLP_test)\n",
    "\n",
    "    \n",
    "print(f\"Training time KAN: {total_train_time_KAN:.2f} seconds\")\n",
    "print(f\"Training time MLP: {total_train_time_MLP:.2f} seconds\")\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs_EEG=mne.concatenate_epochs(data_EEG(subjects=subjects, train=False))\n",
    "df = epochs_EEG.to_data_frame().drop(columns=['time','epoch'])\n",
    "X = df.drop(columns=['condition']).values\n",
    "y = df.condition.values\n",
    "y = le.fit_transform(y)\n",
    "X = torch.from_numpy(X).double()\n",
    " \n",
    "pred_start_time_KAN = time.time()\n",
    "output_KAN= kan_model(X)\n",
    "pred_time_KAN = time.time() - pred_start_time_KAN\n",
    "\n",
    "\n",
    "pred_start_time_MLP = time.time()\n",
    "output_MLP= mlp_model(X)\n",
    "pred_time_MLP = time.time() - pred_start_time_KAN\n",
    "\n",
    "metrics_KAN['pred']=torch.argmax(output_KAN, dim=1).numpy()\n",
    "metrics_KAN['real']=y\n",
    "metrics_KAN['time']=[total_train_time_KAN, pred_time_KAN]\n",
    "\n",
    "filename = 'two_class_RAW_KAN_'+str(subjects[0])+\".json\"\n",
    "export_json(metrics_KAN,filename)\n",
    "\n",
    "\n",
    "metrics_MLP['pred']=torch.argmax(output_MLP, dim=1).numpy()\n",
    "metrics_MLP['real']=y\n",
    "metrics_MLP['time']=[float(total_train_time_MLP), float(pred_time_MLP)]\n",
    "\n",
    "filename = 'two_class_RAW_MLP_'+str(subjects[0])+\".json\"\n",
    "export_json(metrics_MLP,filename)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1> MULTICLASS - RAW </h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.188 | reg: 14.578 : 100%|████████████| 50/50 [00:16<00:00,  3.12it/s]\n",
      "train acc: 0.688 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 48.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.375 | reg: 16.132 : 100%|████████████| 50/50 [00:16<00:00,  3.12it/s]\n",
      "train acc: 0.812 | test acc: 0.062 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.250 | reg: 14.429 : 100%|████████████| 50/50 [00:15<00:00,  3.18it/s]\n",
      "train acc: 0.375 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.062 | reg: 18.184 : 100%|████████████| 50/50 [00:15<00:00,  3.17it/s]\n",
      "train acc: 0.812 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.71it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.500 | test acc: 0.188 | reg: 21.343 : 100%|████████████| 50/50 [00:15<00:00,  3.18it/s]\n",
      "train acc: 0.312 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 45.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.250 | reg: 19.031 : 100%|████████████| 50/50 [00:15<00:00,  3.17it/s]\n",
      "train acc: 0.625 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.125 | reg: 16.714 : 100%|████████████| 50/50 [00:15<00:00,  3.16it/s]\n",
      "train acc: 0.312 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.312 | reg: 15.763 : 100%|████████████| 50/50 [00:15<00:00,  3.18it/s]\n",
      "train acc: 0.250 | test acc: 0.062 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.250 | reg: 17.803 : 100%|████████████| 50/50 [00:15<00:00,  3.16it/s]\n",
      "train acc: 0.500 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.250 | reg: 16.893 : 100%|████████████| 50/50 [00:15<00:00,  3.19it/s]\n",
      "train acc: 0.562 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.312 | reg: 17.321 : 100%|████████████| 50/50 [00:15<00:00,  3.17it/s]\n",
      "train acc: 0.500 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.125 | reg: 19.137 : 100%|████████████| 50/50 [00:15<00:00,  3.21it/s]\n",
      "train acc: 0.562 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 44.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.375 | test acc: 0.125 | reg: 16.411 : 100%|████████████| 50/50 [00:15<00:00,  3.17it/s]\n",
      "train acc: 0.250 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 49.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.438 | reg: 16.556 : 100%|████████████| 50/50 [00:16<00:00,  3.12it/s]\n",
      "train acc: 0.500 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.250 | reg: 14.538 : 100%|████████████| 50/50 [00:15<00:00,  3.18it/s]\n",
      "train acc: 0.500 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.062 | reg: 18.756 : 100%|████████████| 50/50 [00:15<00:00,  3.16it/s]\n",
      "train acc: 0.438 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.188 | reg: 18.852 : 100%|████████████| 50/50 [00:16<00:00,  3.09it/s]\n",
      "train acc: 0.312 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 48.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.188 | reg: 17.581 : 100%|████████████| 50/50 [00:16<00:00,  3.09it/s]\n",
      "train acc: 0.188 | test acc: 0.625 : 100%|██████████████████████████| 50/50 [00:01<00:00, 48.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.500 | test acc: 0.188 | reg: 14.763 : 100%|████████████| 50/50 [00:15<00:00,  3.19it/s]\n",
      "train acc: 0.375 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.188 | reg: 18.664 : 100%|████████████| 50/50 [00:15<00:00,  3.15it/s]\n",
      "train acc: 0.625 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.250 | reg: 17.181 : 100%|████████████| 50/50 [00:15<00:00,  3.16it/s]\n",
      "train acc: 0.562 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 45.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.375 | reg: 16.120 : 100%|████████████| 50/50 [00:15<00:00,  3.15it/s]\n",
      "train acc: 0.625 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 48.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.312 | reg: 17.138 : 100%|████████████| 50/50 [00:15<00:00,  3.14it/s]\n",
      "train acc: 0.375 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.188 | reg: 16.693 : 100%|████████████| 50/50 [00:15<00:00,  3.18it/s]\n",
      "train acc: 0.812 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.500 | test acc: 0.188 | reg: 16.171 : 100%|████████████| 50/50 [00:15<00:00,  3.14it/s]\n",
      "train acc: 0.562 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.500 | test acc: 0.312 | reg: 17.419 : 100%|████████████| 50/50 [00:22<00:00,  2.24it/s]\n",
      "train acc: 0.625 | test acc: 0.062 : 100%|██████████████████████████| 50/50 [00:01<00:00, 35.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.375 | reg: 14.968 : 100%|████████████| 50/50 [00:22<00:00,  2.18it/s]\n",
      "train acc: 0.562 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 30.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.062 | reg: 16.483 : 100%|████████████| 50/50 [00:22<00:00,  2.20it/s]\n",
      "train acc: 0.562 | test acc: 0.000 : 100%|██████████████████████████| 50/50 [00:01<00:00, 31.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.438 | test acc: 0.250 | reg: 16.420 : 100%|████████████| 50/50 [00:23<00:00,  2.10it/s]\n",
      "train acc: 0.438 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 36.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.250 | reg: 14.364 : 100%|████████████| 50/50 [00:23<00:00,  2.16it/s]\n",
      "train acc: 0.375 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 35.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.125 | reg: 17.177 : 100%|████████████| 50/50 [00:21<00:00,  2.30it/s]\n",
      "train acc: 0.438 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:01<00:00, 38.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.125 | reg: 13.675 : 100%|████████████| 50/50 [00:22<00:00,  2.26it/s]\n",
      "train acc: 0.688 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 32.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.188 | reg: 14.745 : 100%|████████████| 50/50 [00:21<00:00,  2.28it/s]\n",
      "train acc: 0.375 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 34.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.250 | reg: 14.750 : 100%|████████████| 50/50 [00:22<00:00,  2.26it/s]\n",
      "train acc: 0.562 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 37.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.375 | reg: 20.436 : 100%|████████████| 50/50 [00:22<00:00,  2.25it/s]\n",
      "train acc: 0.250 | test acc: 0.125 : 100%|██████████████████████████| 50/50 [00:01<00:00, 28.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.438 | reg: 17.365 : 100%|████████████| 50/50 [00:22<00:00,  2.25it/s]\n",
      "train acc: 0.625 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 28.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.125 | reg: 15.430 : 100%|████████████| 50/50 [00:22<00:00,  2.20it/s]\n",
      "train acc: 0.688 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 32.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.062 | reg: 18.574 : 100%|████████████| 50/50 [00:22<00:00,  2.22it/s]\n",
      "train acc: 0.500 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:02<00:00, 24.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.250 | reg: 16.787 : 100%|████████████| 50/50 [00:22<00:00,  2.22it/s]\n",
      "train acc: 0.500 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 33.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.375 | reg: 14.994 : 100%|████████████| 50/50 [00:22<00:00,  2.27it/s]\n",
      "train acc: 0.500 | test acc: 0.125 : 100%|██████████████████████████| 50/50 [00:01<00:00, 36.78it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.438 | reg: 16.633 : 100%|████████████| 50/50 [00:22<00:00,  2.22it/s]\n",
      "train acc: 0.438 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 33.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.188 | reg: 15.762 : 100%|████████████| 50/50 [00:23<00:00,  2.15it/s]\n",
      "train acc: 0.438 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 37.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.250 | reg: 19.172 : 100%|████████████| 50/50 [00:23<00:00,  2.16it/s]\n",
      "train acc: 0.312 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 33.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.188 | reg: 19.031 : 100%|████████████| 50/50 [00:23<00:00,  2.15it/s]\n",
      "train acc: 0.250 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 34.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.062 | reg: 16.123 : 100%|████████████| 50/50 [00:23<00:00,  2.14it/s]\n",
      "train acc: 0.625 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 36.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.250 | reg: 14.780 : 100%|████████████| 50/50 [00:24<00:00,  2.05it/s]\n",
      "train acc: 0.625 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 37.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.250 | reg: 15.981 : 100%|████████████| 50/50 [00:21<00:00,  2.30it/s]\n",
      "train acc: 0.312 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:01<00:00, 40.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.250 | reg: 15.473 : 100%|████████████| 50/50 [00:21<00:00,  2.28it/s]\n",
      "train acc: 0.250 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 27.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.312 | test acc: 0.188 | reg: 16.665 : 100%|████████████| 50/50 [00:22<00:00,  2.22it/s]\n",
      "train acc: 0.500 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 32.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.250 | reg: 15.487 : 100%|████████████| 50/50 [00:22<00:00,  2.25it/s]\n",
      "train acc: 0.375 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 32.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.375 | reg: 20.592 : 100%|████████████| 50/50 [00:22<00:00,  2.24it/s]\n",
      "train acc: 0.188 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 30.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.125 | reg: 18.797 : 100%|████████████| 50/50 [00:22<00:00,  2.19it/s]\n",
      "train acc: 0.438 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 34.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.250 | test acc: 0.438 | reg: 14.410 : 100%|████████████| 50/50 [00:22<00:00,  2.22it/s]\n",
      "train acc: 0.562 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:02<00:00, 24.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.312 | reg: 14.691 : 100%|████████████| 50/50 [00:22<00:00,  2.21it/s]\n",
      "train acc: 0.625 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 34.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.438 | test acc: 0.250 | reg: 15.317 : 100%|████████████| 50/50 [00:22<00:00,  2.25it/s]\n",
      "train acc: 0.500 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 38.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.438 | reg: 17.207 : 100%|████████████| 50/50 [00:22<00:00,  2.25it/s]\n",
      "train acc: 0.312 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 30.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.375 | reg: 13.873 : 100%|████████████| 50/50 [00:23<00:00,  2.16it/s]\n",
      "train acc: 0.375 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 33.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.375 | reg: 14.915 : 100%|████████████| 50/50 [00:22<00:00,  2.18it/s]\n",
      "train acc: 0.562 | test acc: 0.125 : 100%|██████████████████████████| 50/50 [00:01<00:00, 29.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.375 | reg: 15.707 : 100%|████████████| 50/50 [00:23<00:00,  2.10it/s]\n",
      "train acc: 0.688 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 35.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.438 | reg: 13.541 : 100%|████████████| 50/50 [00:23<00:00,  2.09it/s]\n",
      "train acc: 0.438 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 37.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.062 | reg: 13.793 : 100%|████████████| 50/50 [00:22<00:00,  2.22it/s]\n",
      "train acc: 0.312 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 32.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.188 | reg: 13.704 : 100%|████████████| 50/50 [00:16<00:00,  3.02it/s]\n",
      "train acc: 0.250 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.71it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.312 | test acc: 0.375 | reg: 14.776 : 100%|████████████| 50/50 [00:16<00:00,  3.11it/s]\n",
      "train acc: 0.375 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.250 | reg: 15.929 : 100%|████████████| 50/50 [00:18<00:00,  2.66it/s]\n",
      "train acc: 0.375 | test acc: 0.062 : 100%|██████████████████████████| 50/50 [00:01<00:00, 48.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.500 | test acc: 0.250 | reg: 15.142 : 100%|████████████| 50/50 [00:16<00:00,  3.10it/s]\n",
      "train acc: 0.375 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 45.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.312 | reg: 13.446 : 100%|████████████| 50/50 [00:15<00:00,  3.13it/s]\n",
      "train acc: 0.500 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.312 | reg: 14.852 : 100%|████████████| 50/50 [00:15<00:00,  3.19it/s]\n",
      "train acc: 0.375 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 48.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.312 | reg: 14.540 : 100%|████████████| 50/50 [00:15<00:00,  3.20it/s]\n",
      "train acc: 0.375 | test acc: 0.125 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.375 | reg: 15.408 : 100%|████████████| 50/50 [00:15<00:00,  3.14it/s]\n",
      "train acc: 0.625 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 48.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.625 | reg: 14.127 : 100%|████████████| 50/50 [00:20<00:00,  2.49it/s]\n",
      "train acc: 0.250 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 28.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.250 | reg: 14.431 : 100%|████████████| 50/50 [00:23<00:00,  2.17it/s]\n",
      "train acc: 0.625 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 34.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.438 | reg: 15.689 : 100%|████████████| 50/50 [00:22<00:00,  2.25it/s]\n",
      "train acc: 0.375 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 37.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.125 | reg: 16.677 : 100%|████████████| 50/50 [00:22<00:00,  2.26it/s]\n",
      "train acc: 0.188 | test acc: 0.125 : 100%|██████████████████████████| 50/50 [00:01<00:00, 36.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.438 | reg: 14.221 : 100%|████████████| 50/50 [00:22<00:00,  2.19it/s]\n",
      "train acc: 0.562 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 30.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.250 | reg: 16.363 : 100%|████████████| 50/50 [00:22<00:00,  2.21it/s]\n",
      "train acc: 0.312 | test acc: 0.125 : 100%|██████████████████████████| 50/50 [00:01<00:00, 34.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.250 | reg: 13.665 : 100%|████████████| 50/50 [00:23<00:00,  2.14it/s]\n",
      "train acc: 0.312 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:01<00:00, 31.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.500 | test acc: 0.125 | reg: 14.003 : 100%|████████████| 50/50 [00:22<00:00,  2.19it/s]\n",
      "train acc: 0.312 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 36.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.125 | reg: 13.506 : 100%|████████████| 50/50 [00:21<00:00,  2.31it/s]\n",
      "train acc: 0.562 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 35.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.375 | reg: 12.994 : 100%|████████████| 50/50 [00:21<00:00,  2.34it/s]\n",
      "train acc: 0.375 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 35.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.375 | reg: 14.425 : 100%|████████████| 50/50 [00:21<00:00,  2.28it/s]\n",
      "train acc: 0.625 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 32.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.500 | test acc: 0.125 | reg: 13.356 : 100%|████████████| 50/50 [00:21<00:00,  2.28it/s]\n",
      "train acc: 0.375 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 35.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.250 | reg: 14.840 : 100%|████████████| 50/50 [00:22<00:00,  2.23it/s]\n",
      "train acc: 0.375 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 31.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.125 | reg: 15.524 : 100%|████████████| 50/50 [00:21<00:00,  2.33it/s]\n",
      "train acc: 0.375 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 30.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.250 | reg: 13.460 : 100%|████████████| 50/50 [00:23<00:00,  2.16it/s]\n",
      "train acc: 0.500 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 38.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.500 | test acc: 0.125 | reg: 14.969 : 100%|████████████| 50/50 [00:22<00:00,  2.20it/s]\n",
      "train acc: 0.562 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 27.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.312 | reg: 14.755 : 100%|████████████| 50/50 [00:22<00:00,  2.26it/s]\n",
      "train acc: 0.250 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 35.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.250 | reg: 14.870 : 100%|████████████| 50/50 [00:22<00:00,  2.24it/s]\n",
      "train acc: 0.438 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 33.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.375 | reg: 16.958 : 100%|████████████| 50/50 [00:22<00:00,  2.27it/s]\n",
      "train acc: 0.438 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 36.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.500 | test acc: 0.375 | reg: 13.819 : 100%|████████████| 50/50 [00:23<00:00,  2.14it/s]\n",
      "train acc: 0.375 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 37.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.438 | reg: 14.486 : 100%|████████████| 50/50 [00:23<00:00,  2.16it/s]\n",
      "train acc: 0.438 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 33.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.500 | test acc: 0.188 | reg: 13.837 : 100%|████████████| 50/50 [00:22<00:00,  2.20it/s]\n",
      "train acc: 0.625 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 33.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.250 | reg: 14.037 : 100%|████████████| 50/50 [00:23<00:00,  2.17it/s]\n",
      "train acc: 0.375 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 39.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.312 | reg: 12.800 : 100%|████████████| 50/50 [00:23<00:00,  2.15it/s]\n",
      "train acc: 0.250 | test acc: 0.062 : 100%|██████████████████████████| 50/50 [00:01<00:00, 33.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.250 | reg: 15.051 : 100%|████████████| 50/50 [00:22<00:00,  2.27it/s]\n",
      "train acc: 0.750 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 30.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.250 | test acc: 0.312 | reg: 12.971 : 100%|████████████| 50/50 [00:21<00:00,  2.30it/s]\n",
      "train acc: 0.250 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 33.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.438 | reg: 16.069 : 100%|████████████| 50/50 [00:22<00:00,  2.26it/s]\n",
      "train acc: 0.500 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 30.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.250 | reg: 13.879 : 100%|████████████| 50/50 [00:20<00:00,  2.47it/s]\n",
      "train acc: 0.375 | test acc: 0.062 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.79it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.125 | reg: 14.716 : 100%|████████████| 50/50 [00:19<00:00,  2.57it/s]\n",
      "train acc: 0.438 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 29.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.312 | reg: 13.949 : 100%|████████████| 50/50 [00:23<00:00,  2.17it/s]\n",
      "train acc: 0.438 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 28.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.125 | reg: 14.031 : 100%|████████████| 50/50 [00:23<00:00,  2.14it/s]\n",
      "train acc: 0.625 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 32.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.062 | reg: 15.943 : 100%|████████████| 50/50 [00:23<00:00,  2.14it/s]\n",
      "train acc: 0.438 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 31.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.500 | test acc: 0.375 | reg: 14.055 : 100%|████████████| 50/50 [00:20<00:00,  2.39it/s]\n",
      "train acc: 0.625 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.250 | reg: 12.246 : 100%|████████████| 50/50 [00:15<00:00,  3.21it/s]\n",
      "train acc: 0.438 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.250 | reg: 14.576 : 100%|████████████| 50/50 [00:15<00:00,  3.22it/s]\n",
      "train acc: 0.375 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 45.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.062 | reg: 14.260 : 100%|████████████| 50/50 [00:15<00:00,  3.18it/s]\n",
      "train acc: 0.562 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.312 | reg: 14.142 : 100%|████████████| 50/50 [00:15<00:00,  3.17it/s]\n",
      "train acc: 0.500 | test acc: 0.062 : 100%|██████████████████████████| 50/50 [00:01<00:00, 48.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.500 | reg: 14.819 : 100%|████████████| 50/50 [00:16<00:00,  2.96it/s]\n",
      "train acc: 0.438 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 36.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.312 | reg: 13.306 : 100%|████████████| 50/50 [00:21<00:00,  2.35it/s]\n",
      "train acc: 0.438 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 45.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.438 | test acc: 0.188 | reg: 16.946 : 100%|████████████| 50/50 [00:20<00:00,  2.45it/s]\n",
      "train acc: 0.500 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.438 | reg: 13.203 : 100%|████████████| 50/50 [00:16<00:00,  3.12it/s]\n",
      "train acc: 0.500 | test acc: 0.062 : 100%|██████████████████████████| 50/50 [00:01<00:00, 43.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.250 | reg: 13.750 : 100%|████████████| 50/50 [00:15<00:00,  3.13it/s]\n",
      "train acc: 0.438 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 49.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.250 | reg: 14.947 : 100%|████████████| 50/50 [00:16<00:00,  3.12it/s]\n",
      "train acc: 0.562 | test acc: 0.062 : 100%|██████████████████████████| 50/50 [00:01<00:00, 49.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.188 | reg: 15.187 : 100%|████████████| 50/50 [00:16<00:00,  3.11it/s]\n",
      "train acc: 0.250 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 49.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.062 | reg: 15.537 : 100%|████████████| 50/50 [00:16<00:00,  3.08it/s]\n",
      "train acc: 0.062 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.188 | reg: 16.048 : 100%|████████████| 50/50 [00:16<00:00,  3.11it/s]\n",
      "train acc: 0.250 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.125 | reg: 18.212 : 100%|████████████| 50/50 [00:16<00:00,  3.11it/s]\n",
      "train acc: 0.500 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 45.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.250 | reg: 14.974 : 100%|████████████| 50/50 [00:15<00:00,  3.19it/s]\n",
      "train acc: 0.312 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 48.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.125 | reg: 15.810 : 100%|████████████| 50/50 [00:16<00:00,  3.12it/s]\n",
      "train acc: 0.625 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 45.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.125 | reg: 16.134 : 100%|████████████| 50/50 [00:15<00:00,  3.16it/s]\n",
      "train acc: 0.438 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.000 | reg: 13.491 : 100%|████████████| 50/50 [00:15<00:00,  3.15it/s]\n",
      "train acc: 0.188 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:01<00:00, 49.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.250 | reg: 14.099 : 100%|████████████| 50/50 [00:15<00:00,  3.15it/s]\n",
      "train acc: 0.438 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:01<00:00, 48.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.375 | reg: 16.285 : 100%|████████████| 50/50 [00:15<00:00,  3.19it/s]\n",
      "train acc: 0.188 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 45.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.188 | reg: 14.692 : 100%|████████████| 50/50 [00:15<00:00,  3.19it/s]\n",
      "train acc: 0.688 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.438 | test acc: 0.438 | reg: 14.019 : 100%|████████████| 50/50 [00:15<00:00,  3.17it/s]\n",
      "train acc: 0.438 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 45.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.188 | reg: 15.381 : 100%|████████████| 50/50 [00:15<00:00,  3.16it/s]\n",
      "train acc: 0.312 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 48.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.438 | reg: 15.747 : 100%|████████████| 50/50 [00:15<00:00,  3.17it/s]\n",
      "train acc: 0.438 | test acc: 0.125 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.312 | reg: 16.297 : 100%|████████████| 50/50 [00:15<00:00,  3.16it/s]\n",
      "train acc: 0.500 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 45.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.250 | reg: 14.655 : 100%|████████████| 50/50 [00:15<00:00,  3.15it/s]\n",
      "train acc: 0.438 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.438 | test acc: 0.312 | reg: 16.417 : 100%|████████████| 50/50 [00:16<00:00,  3.08it/s]\n",
      "train acc: 0.312 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 45.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.250 | reg: 18.594 : 100%|████████████| 50/50 [00:15<00:00,  3.17it/s]\n",
      "train acc: 0.500 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.250 | reg: 14.872 : 100%|████████████| 50/50 [00:15<00:00,  3.17it/s]\n",
      "train acc: 0.500 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 45.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.250 | reg: 13.089 : 100%|████████████| 50/50 [00:16<00:00,  3.12it/s]\n",
      "train acc: 0.438 | test acc: 0.062 : 100%|██████████████████████████| 50/50 [00:01<00:00, 48.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.125 | reg: 14.633 : 100%|████████████| 50/50 [00:15<00:00,  3.16it/s]\n",
      "train acc: 0.312 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 45.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.062 | reg: 16.660 : 100%|████████████| 50/50 [00:16<00:00,  3.10it/s]\n",
      "train acc: 0.312 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.188 | reg: 14.942 : 100%|████████████| 50/50 [00:16<00:00,  3.10it/s]\n",
      "train acc: 0.438 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.312 | reg: 20.465 : 100%|████████████| 50/50 [00:16<00:00,  3.07it/s]\n",
      "train acc: 0.625 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:01<00:00, 48.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.312 | reg: 14.046 : 100%|████████████| 50/50 [00:16<00:00,  3.09it/s]\n",
      "train acc: 0.438 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.312 | reg: 14.524 : 100%|████████████| 50/50 [00:16<00:00,  3.11it/s]\n",
      "train acc: 0.438 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.125 | reg: 14.957 : 100%|████████████| 50/50 [00:16<00:00,  3.02it/s]\n",
      "train acc: 0.500 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 43.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.438 | reg: 15.087 : 100%|████████████| 50/50 [00:16<00:00,  3.08it/s]\n",
      "train acc: 0.438 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.312 | reg: 14.793 : 100%|████████████| 50/50 [00:16<00:00,  3.11it/s]\n",
      "train acc: 0.500 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.250 | reg: 15.347 : 100%|████████████| 50/50 [00:15<00:00,  3.13it/s]\n",
      "train acc: 0.250 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 44.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.312 | reg: 14.793 : 100%|████████████| 50/50 [00:16<00:00,  3.12it/s]\n",
      "train acc: 0.312 | test acc: 0.125 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.250 | reg: 14.399 : 100%|████████████| 50/50 [00:15<00:00,  3.15it/s]\n",
      "train acc: 0.375 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 45.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.188 | reg: 16.346 : 100%|████████████| 50/50 [00:15<00:00,  3.18it/s]\n",
      "train acc: 0.375 | test acc: 0.125 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.312 | reg: 15.250 : 100%|████████████| 50/50 [00:15<00:00,  3.13it/s]\n",
      "train acc: 0.312 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 45.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.062 | reg: 15.030 : 100%|████████████| 50/50 [00:15<00:00,  3.13it/s]\n",
      "train acc: 0.438 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 48.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.312 | reg: 14.439 : 100%|████████████| 50/50 [00:15<00:00,  3.17it/s]\n",
      "train acc: 0.250 | test acc: 0.125 : 100%|██████████████████████████| 50/50 [00:01<00:00, 48.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.250 | reg: 13.489 : 100%|████████████| 50/50 [00:15<00:00,  3.19it/s]\n",
      "train acc: 0.250 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 44.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.250 | reg: 14.664 : 100%|████████████| 50/50 [00:16<00:00,  3.12it/s]\n",
      "train acc: 0.562 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.250 | reg: 13.583 : 100%|████████████| 50/50 [00:15<00:00,  3.13it/s]\n",
      "train acc: 0.500 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 45.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.312 | reg: 15.799 : 100%|████████████| 50/50 [00:16<00:00,  3.11it/s]\n",
      "train acc: 0.625 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.125 | reg: 14.964 : 100%|████████████| 50/50 [00:15<00:00,  3.13it/s]\n",
      "train acc: 0.625 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.250 | reg: 14.606 : 100%|████████████| 50/50 [00:16<00:00,  3.11it/s]\n",
      "train acc: 0.625 | test acc: 0.125 : 100%|██████████████████████████| 50/50 [00:01<00:00, 44.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.250 | reg: 14.377 : 100%|████████████| 50/50 [00:16<00:00,  3.11it/s]\n",
      "train acc: 0.312 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 44.68it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.312 | reg: 14.831 : 100%|████████████| 50/50 [00:16<00:00,  3.09it/s]\n",
      "train acc: 0.562 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 43.79it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.312 | reg: 15.396 : 100%|████████████| 50/50 [00:16<00:00,  3.07it/s]\n",
      "train acc: 0.500 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 45.71it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.125 | reg: 14.345 : 100%|████████████| 50/50 [00:16<00:00,  3.05it/s]\n",
      "train acc: 0.375 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:01<00:00, 44.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.312 | reg: 15.727 : 100%|████████████| 50/50 [00:16<00:00,  3.11it/s]\n",
      "train acc: 0.438 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 43.68it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.188 | reg: 16.048 : 100%|████████████| 50/50 [00:16<00:00,  3.07it/s]\n",
      "train acc: 0.375 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.500 | test acc: 0.375 | reg: 15.380 : 100%|████████████| 50/50 [00:22<00:00,  2.21it/s]\n",
      "train acc: 0.312 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 33.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.500 | test acc: 0.125 | reg: 15.625 : 100%|████████████| 50/50 [00:23<00:00,  2.15it/s]\n",
      "train acc: 0.312 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 31.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.250 | reg: 16.008 : 100%|████████████| 50/50 [00:26<00:00,  1.91it/s]\n",
      "train acc: 0.312 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 30.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.250 | reg: 16.271 : 100%|████████████| 50/50 [00:25<00:00,  1.99it/s]\n",
      "train acc: 0.625 | test acc: 0.125 : 100%|██████████████████████████| 50/50 [00:01<00:00, 34.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.312 | reg: 14.539 : 100%|████████████| 50/50 [00:23<00:00,  2.13it/s]\n",
      "train acc: 0.500 | test acc: 0.125 : 100%|██████████████████████████| 50/50 [00:01<00:00, 33.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.250 | reg: 15.688 : 100%|████████████| 50/50 [00:23<00:00,  2.14it/s]\n",
      "train acc: 0.125 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 30.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.188 | reg: 17.002 : 100%|████████████| 50/50 [00:24<00:00,  2.07it/s]\n",
      "train acc: 0.188 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 31.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.250 | reg: 15.125 : 100%|████████████| 50/50 [00:26<00:00,  1.88it/s]\n",
      "train acc: 0.438 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 38.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.062 | reg: 16.934 : 100%|████████████| 50/50 [00:21<00:00,  2.31it/s]\n",
      "train acc: 0.250 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:01<00:00, 33.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.250 | reg: 15.895 : 100%|████████████| 50/50 [00:23<00:00,  2.11it/s]\n",
      "train acc: 0.625 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:01<00:00, 31.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.125 | reg: 15.180 : 100%|████████████| 50/50 [00:23<00:00,  2.11it/s]\n",
      "train acc: 0.375 | test acc: 0.500 : 100%|██████████████████████████| 50/50 [00:01<00:00, 36.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.438 | test acc: 0.312 | reg: 14.963 : 100%|████████████| 50/50 [00:23<00:00,  2.16it/s]\n",
      "train acc: 0.375 | test acc: 0.000 : 100%|██████████████████████████| 50/50 [00:01<00:00, 32.79it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.500 | test acc: 0.375 | reg: 15.667 : 100%|████████████| 50/50 [00:24<00:00,  2.01it/s]\n",
      "train acc: 0.375 | test acc: 0.438 : 100%|██████████████████████████| 50/50 [00:01<00:00, 33.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.312 | reg: 14.471 : 100%|████████████| 50/50 [00:22<00:00,  2.21it/s]\n",
      "train acc: 0.625 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 34.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.375 | reg: 13.758 : 100%|████████████| 50/50 [00:17<00:00,  2.91it/s]\n",
      "train acc: 0.312 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.68it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.188 | reg: 14.576 : 100%|████████████| 50/50 [00:16<00:00,  2.96it/s]\n",
      "train acc: 0.312 | test acc: 0.062 : 100%|██████████████████████████| 50/50 [00:01<00:00, 45.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.375 | reg: 15.511 : 100%|████████████| 50/50 [00:16<00:00,  3.04it/s]\n",
      "train acc: 0.438 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 38.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.500 | test acc: 0.250 | reg: 15.552 : 100%|████████████| 50/50 [00:16<00:00,  3.08it/s]\n",
      "train acc: 0.312 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.188 | reg: 15.173 : 100%|████████████| 50/50 [00:16<00:00,  3.10it/s]\n",
      "train acc: 0.500 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 43.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.125 | reg: 14.391 : 100%|████████████| 50/50 [00:16<00:00,  3.10it/s]\n",
      "train acc: 0.375 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 44.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.188 | reg: 14.954 : 100%|████████████| 50/50 [00:16<00:00,  3.07it/s]\n",
      "train acc: 0.438 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.938 | test acc: 0.250 | reg: 14.200 : 100%|████████████| 50/50 [00:15<00:00,  3.13it/s]\n",
      "train acc: 0.250 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 45.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.125 | reg: 17.141 : 100%|████████████| 50/50 [00:15<00:00,  3.13it/s]\n",
      "train acc: 0.562 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.375 | reg: 16.344 : 100%|████████████| 50/50 [00:15<00:00,  3.16it/s]\n",
      "train acc: 0.312 | test acc: 0.312 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.750 | test acc: 0.375 | reg: 15.866 : 100%|████████████| 50/50 [00:15<00:00,  3.19it/s]\n",
      "train acc: 0.375 | test acc: 0.125 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.438 | test acc: 0.500 | reg: 14.611 : 100%|████████████| 50/50 [00:15<00:00,  3.17it/s]\n",
      "train acc: 0.438 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.312 | reg: 15.852 : 100%|████████████| 50/50 [00:15<00:00,  3.19it/s]\n",
      "train acc: 0.312 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.188 | reg: 14.686 : 100%|████████████| 50/50 [00:15<00:00,  3.18it/s]\n",
      "train acc: 0.438 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.250 | reg: 15.101 : 100%|████████████| 50/50 [00:16<00:00,  3.11it/s]\n",
      "train acc: 0.500 | test acc: 0.125 : 100%|██████████████████████████| 50/50 [00:01<00:00, 49.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.438 | reg: 14.669 : 100%|████████████| 50/50 [00:15<00:00,  3.16it/s]\n",
      "train acc: 0.375 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.062 | reg: 14.991 : 100%|████████████| 50/50 [00:15<00:00,  3.16it/s]\n",
      "train acc: 0.500 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 48.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.188 | reg: 14.272 : 100%|████████████| 50/50 [00:16<00:00,  3.12it/s]\n",
      "train acc: 0.500 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.438 | reg: 17.942 : 100%|████████████| 50/50 [00:16<00:00,  3.10it/s]\n",
      "train acc: 0.438 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.688 | test acc: 0.500 | reg: 13.699 : 100%|████████████| 50/50 [00:15<00:00,  3.18it/s]\n",
      "train acc: 0.500 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 45.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.562 | test acc: 0.125 | reg: 14.502 : 100%|████████████| 50/50 [00:15<00:00,  3.15it/s]\n",
      "train acc: 0.688 | test acc: 0.188 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.812 | test acc: 0.188 | reg: 15.905 : 100%|████████████| 50/50 [00:15<00:00,  3.15it/s]\n",
      "train acc: 0.250 | test acc: 0.000 : 100%|██████████████████████████| 50/50 [00:01<00:00, 44.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.250 | reg: 14.380 : 100%|████████████| 50/50 [00:15<00:00,  3.14it/s]\n",
      "train acc: 0.250 | test acc: 0.000 : 100%|██████████████████████████| 50/50 [00:01<00:00, 48.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.875 | test acc: 0.250 | reg: 15.522 : 100%|████████████| 50/50 [00:16<00:00,  3.10it/s]\n",
      "train acc: 0.312 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 47.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.438 | test acc: 0.250 | reg: 17.146 : 100%|████████████| 50/50 [00:16<00:00,  3.09it/s]\n",
      "train acc: 0.375 | test acc: 0.250 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.625 | test acc: 0.125 | reg: 18.670 : 100%|████████████| 50/50 [00:16<00:00,  3.09it/s]\n",
      "train acc: 0.625 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:01<00:00, 46.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training time KAN: 3766.46 seconds\n",
      "Training time MLP: 250.58 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "epochs_EEG=mne.concatenate_epochs(data_EEG(subjects=subjects, multiclass=True))\n",
    "df = epochs_EEG.to_data_frame().drop(columns=['time','epoch'])\n",
    "X = df.drop(columns=['condition']).values\n",
    "y = df.condition.values\n",
    "y = le.fit_transform(y)\n",
    "\n",
    "X = torch.from_numpy(X).double()\n",
    "y  = torch.from_numpy(y).long()\n",
    "\n",
    "\n",
    "input_layer = X.shape[1]\n",
    "output_layer = len(np.unique(y))\n",
    "\n",
    "\n",
    "kan_model  = creating_models(input_layer, output_layer)\n",
    "kan_optimizer = LBFGS(kan_model.parameters(), lr=lr) \n",
    "\n",
    "mlp_model = MLP(input_layer, hidden_layers, output_layer).to(device)\n",
    "\n",
    "for param in mlp_model.parameters():\n",
    "    param.data = param.data.double()\n",
    "mlp_optimizer = LBFGS(mlp_model.parameters(), lr=lr, history_size=10, line_search_fn=\"strong_wolfe\", tolerance_grad=1e-32, tolerance_change=1e-32, tolerance_ys=1e-32) #(mlp_model.parameters(), lr=lr)\n",
    "\n",
    "\n",
    "metrics_KAN = {}  \n",
    "metrics_KAN['train'] = []\n",
    "metrics_KAN['test'] = []\n",
    "\n",
    "metrics_MLP = {}  \n",
    "metrics_MLP['train'] = []\n",
    "metrics_MLP['test'] = []\n",
    "\n",
    "total_train_time_KAN = 0\n",
    "total_train_time_MLP = 0\n",
    "\n",
    "for fold, (train_index, test_index) in enumerate(kf.split(X)):\n",
    "\n",
    "    X_train, X_test = X[train_index], X[test_index]\n",
    "    y_train, y_test = y[train_index], y[test_index]\n",
    "\n",
    "    dataset={}\n",
    "    dataset['train_input']= X_train\n",
    "    dataset['train_label']= y_train\n",
    "    dataset['test_input']= X_test\n",
    "    dataset['test_label']= y_test\n",
    "\n",
    "    fold_metrics_KAN_train = []\n",
    "    fold_metrics_KAN_test =[]\n",
    "    fold_metrics_MLP_train = []\n",
    "    fold_metrics_MLP_test =[]\n",
    "\n",
    "    for epoch in range(epochs):\n",
    "        print('###### EPOCH ', epoch, \" ############\")\n",
    "\n",
    "        epoch_start_time_KAN = time.time()\n",
    "\n",
    "        results_kan = kan_model.train(dataset, opt=\"LBFGS\", steps=steps, \n",
    "        loss_fn=loss_fn, batch=batch, lr=lr)\n",
    "        \n",
    "        epoch_end_time_KAN = time.time()\n",
    "        epoch_train_KAN = epoch_end_time_KAN - epoch_start_time_KAN\n",
    "        total_train_time_KAN += epoch_train_KAN\n",
    "        \n",
    "        epoch_start_time_MLP = time.time()\n",
    "\n",
    "        fold_metrics_KAN_train.append(np.mean(results_kan['train_acc']))\n",
    "        fold_metrics_KAN_test.append(np.mean(results_kan['test_acc'])) \n",
    "\n",
    "        results_mlp = mlp_model.train(dataset, opt=\"LBFGS\", steps=steps, \n",
    "        loss_fn=loss_fn, batch=batch, lr=lr)\n",
    "\n",
    "        epoch_end_time_MLP = time.time()\n",
    "        epoch_train_MLP = epoch_end_time_MLP - epoch_start_time_MLP\n",
    "        total_train_time_MLP += epoch_train_MLP\n",
    "        \n",
    "        fold_metrics_MLP_train.append(np.mean(results_mlp['train_acc']))\n",
    "        fold_metrics_MLP_test.append(np.mean(results_mlp['test_acc'])) \n",
    "\n",
    "    metrics_KAN['train'].append(fold_metrics_KAN_train)\n",
    "    metrics_KAN['test'].append(fold_metrics_KAN_test)\n",
    "    \n",
    "    metrics_MLP['train'].append(fold_metrics_MLP_train)\n",
    "    metrics_MLP['test'].append(fold_metrics_MLP_test)\n",
    "\n",
    "    \n",
    "print(f\"Training time KAN: {total_train_time_KAN:.2f} seconds\")\n",
    "print(f\"Training time MLP: {total_train_time_MLP:.2f} seconds\")\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs_EEG=mne.concatenate_epochs(data_EEG(subjects=subjects, train=False, multiclass=True))\n",
    "df = epochs_EEG.to_data_frame().drop(columns=['time','epoch'])\n",
    "X = df.drop(columns=['condition']).values\n",
    "y = df.condition.values\n",
    "y = le.fit_transform(y)\n",
    "X = torch.from_numpy(X).double()\n",
    " \n",
    "pred_start_time_KAN = time.time()\n",
    "output_KAN= kan_model(X)\n",
    "pred_time_KAN = time.time() - pred_start_time_KAN\n",
    "\n",
    "\n",
    "pred_start_time_MLP = time.time()\n",
    "output_MLP= mlp_model(X)\n",
    "pred_time_MLP = time.time() - pred_start_time_KAN\n",
    "\n",
    "metrics_KAN['pred']=torch.argmax(output_KAN, dim=1).numpy()\n",
    "metrics_KAN['real']=y\n",
    "metrics_KAN['time']=[total_train_time_KAN, pred_time_KAN]\n",
    "\n",
    "filename = 'multiclass_RAW_KAN_'+str(subjects[0])+\".json\"\n",
    "export_json(metrics_KAN,filename)\n",
    "\n",
    "\n",
    "metrics_MLP['pred']=torch.argmax(output_MLP, dim=1).numpy()\n",
    "metrics_MLP['real']=y\n",
    "metrics_MLP['time']=[float(total_train_time_MLP), float(pred_time_MLP)]\n",
    "\n",
    "filename = 'multiclass_RAW_MLP_'+str(subjects[0])+\".json\"\n",
    "export_json(metrics_MLP,filename)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1> TWO CLASS - CSP </h1> "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.981 | test acc: 1.000 | reg: 166.931 : 100%|███████████| 50/50 [00:03<00:00, 12.76it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 106.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 399.788 : 100%|███████████| 50/50 [00:04<00:00, 12.34it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 178.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 1157.771 : 100%|██████████| 50/50 [00:04<00:00, 12.45it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 156.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 1109.155 : 100%|██████████| 50/50 [00:03<00:00, 12.75it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 179.79it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 1539.814 : 100%|██████████| 50/50 [00:04<00:00, 12.16it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 178.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 1582.736 : 100%|██████████| 50/50 [00:04<00:00, 12.28it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 178.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 1842.554 : 100%|██████████| 50/50 [00:03<00:00, 12.52it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 178.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 2201.950 : 100%|██████████| 50/50 [00:03<00:00, 12.51it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 175.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 2847.933 : 100%|██████████| 50/50 [00:04<00:00, 12.26it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 180.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 3064.701 : 100%|██████████| 50/50 [00:04<00:00, 12.14it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 176.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 5654.501 : 100%|██████████| 50/50 [00:04<00:00, 12.20it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 178.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 6202.939 : 100%|██████████| 50/50 [00:04<00:00, 12.25it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 178.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 6221.334 : 100%|██████████| 50/50 [00:04<00:00, 12.49it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 180.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7753.438 : 100%|██████████| 50/50 [00:04<00:00, 12.36it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 183.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7675.691 : 100%|██████████| 50/50 [00:02<00:00, 21.42it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 184.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7675.691 : 100%|█████████| 50/50 [00:00<00:00, 101.90it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 171.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7675.691 : 100%|██████████| 50/50 [00:00<00:00, 99.43it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 184.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7675.691 : 100%|█████████| 50/50 [00:00<00:00, 100.02it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 155.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7675.691 : 100%|█████████| 50/50 [00:00<00:00, 100.43it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 173.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7675.691 : 100%|█████████| 50/50 [00:00<00:00, 101.13it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 182.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7738.122 : 100%|██████████| 50/50 [00:03<00:00, 12.68it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 111.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7738.122 : 100%|██████████| 50/50 [00:00<00:00, 89.80it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 280.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7738.122 : 100%|█████████| 50/50 [00:00<00:00, 101.70it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 277.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7738.122 : 100%|█████████| 50/50 [00:00<00:00, 101.08it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 278.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7738.122 : 100%|█████████| 50/50 [00:00<00:00, 100.22it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 278.68it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7738.122 : 100%|██████████| 50/50 [00:00<00:00, 89.43it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 268.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7738.122 : 100%|█████████| 50/50 [00:00<00:00, 100.29it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 278.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7738.122 : 100%|█████████| 50/50 [00:00<00:00, 100.56it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 275.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7738.122 : 100%|██████████| 50/50 [00:00<00:00, 95.99it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 264.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7738.122 : 100%|██████████| 50/50 [00:00<00:00, 90.97it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 281.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7738.122 : 100%|██████████| 50/50 [00:00<00:00, 97.03it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 255.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7738.122 : 100%|██████████| 50/50 [00:00<00:00, 93.65it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 244.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7738.122 : 100%|█████████| 50/50 [00:00<00:00, 104.61it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 279.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7738.122 : 100%|██████████| 50/50 [00:00<00:00, 93.37it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 257.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7738.122 : 100%|█████████| 50/50 [00:00<00:00, 100.38it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 280.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7738.122 : 100%|█████████| 50/50 [00:00<00:00, 102.47it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 275.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7738.122 : 100%|█████████| 50/50 [00:00<00:00, 101.24it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 279.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7738.122 : 100%|██████████| 50/50 [00:00<00:00, 92.02it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 262.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7738.122 : 100%|█████████| 50/50 [00:00<00:00, 102.18it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 283.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7738.122 : 100%|█████████| 50/50 [00:00<00:00, 102.39it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 280.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.750 | reg: 8083.534 : 100%|██████████| 50/50 [00:04<00:00, 12.49it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 113.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.750 | reg: 8083.534 : 100%|██████████| 50/50 [00:00<00:00, 91.48it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 147.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.750 | reg: 8083.534 : 100%|█████████| 50/50 [00:00<00:00, 101.25it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 180.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.750 | reg: 8083.534 : 100%|█████████| 50/50 [00:00<00:00, 101.35it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 122.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.750 | reg: 8083.534 : 100%|██████████| 50/50 [00:00<00:00, 95.65it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 140.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.750 | reg: 8083.534 : 100%|█████████| 50/50 [00:00<00:00, 100.71it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 166.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.750 | reg: 8083.534 : 100%|█████████| 50/50 [00:00<00:00, 101.95it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 156.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.750 | reg: 8083.534 : 100%|█████████| 50/50 [00:00<00:00, 101.20it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 167.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.750 | reg: 8083.534 : 100%|██████████| 50/50 [00:00<00:00, 92.91it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 165.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.750 | reg: 8083.534 : 100%|██████████| 50/50 [00:00<00:00, 99.08it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 158.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.750 | reg: 8083.534 : 100%|█████████| 50/50 [00:00<00:00, 102.65it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 168.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.750 | reg: 8083.534 : 100%|█████████| 50/50 [00:00<00:00, 102.93it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 165.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.750 | reg: 8083.534 : 100%|██████████| 50/50 [00:00<00:00, 88.83it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 163.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.750 | reg: 8083.534 : 100%|█████████| 50/50 [00:00<00:00, 103.08it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 166.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.750 | reg: 8083.534 : 100%|█████████| 50/50 [00:00<00:00, 101.64it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 165.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.750 | reg: 8083.534 : 100%|██████████| 50/50 [00:00<00:00, 98.90it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 143.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.750 | reg: 8083.534 : 100%|█████████| 50/50 [00:00<00:00, 100.40it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 166.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.750 | reg: 8083.534 : 100%|█████████| 50/50 [00:00<00:00, 101.19it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 152.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.750 | reg: 8083.534 : 100%|██████████| 50/50 [00:00<00:00, 96.69it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 166.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.750 | reg: 8083.534 : 100%|██████████| 50/50 [00:00<00:00, 92.90it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 172.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 7491.537 : 100%|██████████| 50/50 [00:04<00:00, 12.27it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 159.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.833 | reg: 7747.354 : 100%|██████████| 50/50 [00:04<00:00, 12.15it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 165.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.833 | reg: 7747.354 : 100%|██████████| 50/50 [00:00<00:00, 90.22it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 159.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.833 | reg: 7747.354 : 100%|█████████| 50/50 [00:00<00:00, 101.45it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 167.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.833 | reg: 7747.354 : 100%|█████████| 50/50 [00:00<00:00, 103.27it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 170.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.833 | reg: 7747.354 : 100%|██████████| 50/50 [00:00<00:00, 99.95it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 167.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.833 | reg: 7747.354 : 100%|██████████| 50/50 [00:00<00:00, 93.50it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 165.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.833 | reg: 7747.354 : 100%|█████████| 50/50 [00:00<00:00, 103.21it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 163.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.833 | reg: 7747.354 : 100%|█████████| 50/50 [00:00<00:00, 101.57it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 166.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.833 | reg: 7747.354 : 100%|█████████| 50/50 [00:00<00:00, 100.25it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 143.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.833 | reg: 7747.354 : 100%|█████████| 50/50 [00:00<00:00, 101.32it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 156.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.833 | reg: 7747.354 : 100%|█████████| 50/50 [00:00<00:00, 100.62it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 165.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.833 | reg: 7747.354 : 100%|█████████| 50/50 [00:00<00:00, 102.26it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 165.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.833 | reg: 7747.354 : 100%|██████████| 50/50 [00:00<00:00, 91.59it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 156.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.833 | reg: 7747.354 : 100%|█████████| 50/50 [00:00<00:00, 102.32it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 169.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.833 | reg: 7747.354 : 100%|█████████| 50/50 [00:00<00:00, 101.85it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 163.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.833 | reg: 7747.354 : 100%|██████████| 50/50 [00:00<00:00, 98.65it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 160.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.833 | reg: 7747.354 : 100%|██████████| 50/50 [00:00<00:00, 91.25it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 167.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.833 | reg: 7747.354 : 100%|█████████| 50/50 [00:00<00:00, 102.18it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 168.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.833 | reg: 7747.354 : 100%|██████████| 50/50 [00:00<00:00, 99.43it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 170.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 6363.345 : 100%|██████████| 50/50 [00:04<00:00, 12.12it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 173.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 7840.841 : 100%|██████████| 50/50 [00:04<00:00, 12.18it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 176.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 7840.841 : 100%|█████████| 50/50 [00:00<00:00, 100.45it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 232.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 7840.841 : 100%|██████████| 50/50 [00:00<00:00, 99.02it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 238.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 7840.841 : 100%|██████████| 50/50 [00:00<00:00, 88.77it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 226.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 7840.841 : 100%|█████████| 50/50 [00:00<00:00, 101.46it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 236.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 7840.841 : 100%|█████████| 50/50 [00:00<00:00, 102.89it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 237.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 7840.841 : 100%|█████████| 50/50 [00:00<00:00, 100.79it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 230.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 7840.841 : 100%|██████████| 50/50 [00:00<00:00, 92.01it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 237.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 7840.841 : 100%|█████████| 50/50 [00:00<00:00, 100.83it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 233.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 7840.841 : 100%|██████████| 50/50 [00:00<00:00, 94.91it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 221.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 7840.841 : 100%|██████████| 50/50 [00:00<00:00, 99.83it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 235.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 7840.841 : 100%|██████████| 50/50 [00:00<00:00, 91.97it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 230.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 7840.841 : 100%|██████████| 50/50 [00:00<00:00, 98.07it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 208.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 7840.841 : 100%|██████████| 50/50 [00:00<00:00, 89.29it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 186.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 7840.841 : 100%|██████████| 50/50 [00:00<00:00, 98.44it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 221.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 7840.841 : 100%|██████████| 50/50 [00:00<00:00, 85.78it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 208.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 7840.841 : 100%|██████████| 50/50 [00:00<00:00, 96.75it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 236.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 7840.841 : 100%|██████████| 50/50 [00:00<00:00, 98.97it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 231.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 7840.841 : 100%|█████████| 50/50 [00:00<00:00, 100.60it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 188.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7956.795 : 100%|██████████| 50/50 [00:03<00:00, 12.55it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 214.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7956.795 : 100%|██████████| 50/50 [00:00<00:00, 97.04it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 228.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7956.795 : 100%|██████████| 50/50 [00:00<00:00, 98.65it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 223.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7956.795 : 100%|██████████| 50/50 [00:00<00:00, 91.84it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 202.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7956.795 : 100%|██████████| 50/50 [00:00<00:00, 98.57it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 230.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7956.795 : 100%|█████████| 50/50 [00:00<00:00, 100.07it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 230.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7956.795 : 100%|██████████| 50/50 [00:00<00:00, 99.82it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 228.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7956.795 : 100%|██████████| 50/50 [00:00<00:00, 86.79it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 224.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7956.795 : 100%|██████████| 50/50 [00:00<00:00, 99.95it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 229.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7956.795 : 100%|█████████| 50/50 [00:00<00:00, 100.95it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 208.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7956.795 : 100%|██████████| 50/50 [00:00<00:00, 96.00it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 231.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7956.795 : 100%|██████████| 50/50 [00:00<00:00, 89.40it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 238.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7956.795 : 100%|██████████| 50/50 [00:00<00:00, 99.28it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 207.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7956.795 : 100%|██████████| 50/50 [00:00<00:00, 97.63it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 238.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7956.795 : 100%|██████████| 50/50 [00:00<00:00, 99.02it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 231.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7956.795 : 100%|██████████| 50/50 [00:00<00:00, 92.76it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 227.48it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7956.795 : 100%|█████████| 50/50 [00:00<00:00, 100.54it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 223.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7956.795 : 100%|█████████| 50/50 [00:00<00:00, 100.83it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 231.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7956.795 : 100%|██████████| 50/50 [00:00<00:00, 99.62it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 217.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 7956.795 : 100%|██████████| 50/50 [00:00<00:00, 90.99it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 220.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8038.264 : 100%|██████████| 50/50 [00:03<00:00, 12.65it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 236.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8038.263 : 100%|██████████| 50/50 [00:00<00:00, 60.78it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 233.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8038.263 : 100%|██████████| 50/50 [00:00<00:00, 58.35it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 219.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8038.263 : 100%|██████████| 50/50 [00:00<00:00, 62.22it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 231.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8038.263 : 100%|██████████| 50/50 [00:00<00:00, 62.81it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 199.68it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8038.263 : 100%|██████████| 50/50 [00:00<00:00, 59.77it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 193.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8038.263 : 100%|██████████| 50/50 [00:00<00:00, 60.24it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 206.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8038.263 : 100%|██████████| 50/50 [00:00<00:00, 60.73it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 235.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8038.263 : 100%|██████████| 50/50 [00:00<00:00, 61.76it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 224.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8038.263 : 100%|██████████| 50/50 [00:00<00:00, 55.72it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 232.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8038.263 : 100%|██████████| 50/50 [00:00<00:00, 63.01it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 226.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8038.263 : 100%|██████████| 50/50 [00:00<00:00, 63.02it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 227.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8038.263 : 100%|██████████| 50/50 [00:00<00:00, 57.85it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 222.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8038.263 : 100%|██████████| 50/50 [00:00<00:00, 62.27it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 234.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8038.263 : 100%|██████████| 50/50 [00:00<00:00, 61.98it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 236.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8038.263 : 100%|██████████| 50/50 [00:00<00:00, 59.36it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 226.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8038.263 : 100%|██████████| 50/50 [00:00<00:00, 62.31it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 231.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8038.263 : 100%|██████████| 50/50 [00:00<00:00, 61.47it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 220.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8038.263 : 100%|██████████| 50/50 [00:00<00:00, 61.96it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 190.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8038.263 : 100%|██████████| 50/50 [00:00<00:00, 62.03it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 219.78it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.667 | reg: 8072.988 : 100%|██████████| 50/50 [00:02<00:00, 20.08it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 142.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.667 | reg: 8072.988 : 100%|██████████| 50/50 [00:00<00:00, 62.78it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 115.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.667 | reg: 8072.988 : 100%|██████████| 50/50 [00:00<00:00, 61.14it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 128.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.667 | reg: 8072.988 : 100%|██████████| 50/50 [00:00<00:00, 62.46it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 126.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.667 | reg: 8072.988 : 100%|██████████| 50/50 [00:00<00:00, 58.18it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 125.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.667 | reg: 8072.988 : 100%|██████████| 50/50 [00:00<00:00, 62.07it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 122.78it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.667 | reg: 8072.988 : 100%|██████████| 50/50 [00:00<00:00, 62.08it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 125.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.667 | reg: 8072.988 : 100%|██████████| 50/50 [00:00<00:00, 58.31it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 119.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.667 | reg: 8072.988 : 100%|██████████| 50/50 [00:00<00:00, 62.03it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 126.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.667 | reg: 8072.988 : 100%|██████████| 50/50 [00:00<00:00, 63.02it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 125.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.667 | reg: 8072.988 : 100%|██████████| 50/50 [00:00<00:00, 60.41it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 128.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.667 | reg: 8072.988 : 100%|██████████| 50/50 [00:00<00:00, 62.31it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 125.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.667 | reg: 8072.988 : 100%|██████████| 50/50 [00:00<00:00, 62.93it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 127.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.667 | reg: 8072.988 : 100%|██████████| 50/50 [00:00<00:00, 57.57it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 128.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.667 | reg: 8072.988 : 100%|██████████| 50/50 [00:00<00:00, 61.65it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 128.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.667 | reg: 8072.988 : 100%|██████████| 50/50 [00:00<00:00, 62.26it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 123.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.667 | reg: 8072.988 : 100%|██████████| 50/50 [00:00<00:00, 59.58it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 124.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.667 | reg: 8072.988 : 100%|██████████| 50/50 [00:00<00:00, 62.17it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 127.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.667 | reg: 8072.988 : 100%|██████████| 50/50 [00:00<00:00, 63.29it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 130.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.667 | reg: 8072.988 : 100%|██████████| 50/50 [00:00<00:00, 59.77it/s]\n",
      "train acc: 0.981 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 129.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 8756.664 : 100%|██████████| 50/50 [00:04<00:00, 12.43it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 187.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 8756.664 : 100%|█████████| 50/50 [00:00<00:00, 100.79it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 527.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 8756.664 : 100%|██████████| 50/50 [00:00<00:00, 90.70it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 246.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 8756.664 : 100%|██████████| 50/50 [00:00<00:00, 96.87it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 319.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 8756.664 : 100%|██████████| 50/50 [00:00<00:00, 98.50it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 331.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 8756.664 : 100%|██████████| 50/50 [00:00<00:00, 98.74it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 326.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 8756.664 : 100%|██████████| 50/50 [00:00<00:00, 96.81it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 231.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 8756.664 : 100%|██████████| 50/50 [00:00<00:00, 99.08it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 328.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 8756.664 : 100%|█████████| 50/50 [00:00<00:00, 100.95it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 326.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 8756.664 : 100%|█████████| 50/50 [00:00<00:00, 100.56it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 296.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 8756.664 : 100%|█████████| 50/50 [00:00<00:00, 100.14it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 326.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 8756.664 : 100%|██████████| 50/50 [00:00<00:00, 89.32it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 316.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 8756.664 : 100%|██████████| 50/50 [00:00<00:00, 99.76it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 322.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 8756.664 : 100%|██████████| 50/50 [00:00<00:00, 94.29it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 331.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 8756.664 : 100%|█████████| 50/50 [00:00<00:00, 101.13it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 326.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 8756.664 : 100%|██████████| 50/50 [00:00<00:00, 92.69it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 325.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 8756.664 : 100%|██████████| 50/50 [00:00<00:00, 95.93it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 319.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 8756.664 : 100%|██████████| 50/50 [00:00<00:00, 96.93it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 326.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 8756.664 : 100%|█████████| 50/50 [00:00<00:00, 100.63it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 332.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 1.000 | reg: 8756.664 : 100%|██████████| 50/50 [00:00<00:00, 89.69it/s]\n",
      "train acc: 0.981 | test acc: 1.000 : 100%|█████████████████████████| 50/50 [00:00<00:00, 290.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8518.921 : 100%|██████████| 50/50 [00:02<00:00, 21.52it/s]\n",
      "train acc: 1.000 | test acc: 0.917 : 100%|██████████████████████████| 50/50 [00:00<00:00, 91.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8518.921 : 100%|██████████| 50/50 [00:00<00:00, 62.44it/s]\n",
      "train acc: 1.000 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 342.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8518.921 : 100%|██████████| 50/50 [00:00<00:00, 59.35it/s]\n",
      "train acc: 1.000 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 344.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8518.921 : 100%|██████████| 50/50 [00:00<00:00, 62.47it/s]\n",
      "train acc: 1.000 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 353.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8518.921 : 100%|██████████| 50/50 [00:00<00:00, 62.75it/s]\n",
      "train acc: 1.000 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 330.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8518.921 : 100%|██████████| 50/50 [00:00<00:00, 63.26it/s]\n",
      "train acc: 1.000 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 342.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8518.921 : 100%|██████████| 50/50 [00:00<00:00, 59.79it/s]\n",
      "train acc: 1.000 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 343.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8518.921 : 100%|██████████| 50/50 [00:00<00:00, 62.90it/s]\n",
      "train acc: 1.000 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 348.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8518.921 : 100%|██████████| 50/50 [00:00<00:00, 63.62it/s]\n",
      "train acc: 1.000 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 344.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8518.921 : 100%|██████████| 50/50 [00:00<00:00, 55.76it/s]\n",
      "train acc: 1.000 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 298.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8518.921 : 100%|██████████| 50/50 [00:00<00:00, 62.49it/s]\n",
      "train acc: 1.000 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 340.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8518.921 : 100%|██████████| 50/50 [00:00<00:00, 61.89it/s]\n",
      "train acc: 1.000 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 302.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8518.921 : 100%|██████████| 50/50 [00:00<00:00, 54.96it/s]\n",
      "train acc: 1.000 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 315.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8518.921 : 100%|██████████| 50/50 [00:00<00:00, 61.33it/s]\n",
      "train acc: 1.000 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 335.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8518.921 : 100%|██████████| 50/50 [00:00<00:00, 61.27it/s]\n",
      "train acc: 1.000 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 344.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8518.921 : 100%|██████████| 50/50 [00:00<00:00, 63.29it/s]\n",
      "train acc: 1.000 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 352.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8518.921 : 100%|██████████| 50/50 [00:00<00:00, 60.40it/s]\n",
      "train acc: 1.000 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 342.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8518.921 : 100%|██████████| 50/50 [00:00<00:00, 63.88it/s]\n",
      "train acc: 1.000 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 342.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8518.921 : 100%|██████████| 50/50 [00:00<00:00, 64.08it/s]\n",
      "train acc: 1.000 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 337.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 1.000 | test acc: 0.917 | reg: 8518.921 : 100%|██████████| 50/50 [00:00<00:00, 59.11it/s]\n",
      "train acc: 1.000 | test acc: 0.917 : 100%|█████████████████████████| 50/50 [00:00<00:00, 341.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training time KAN: 206.77 seconds\n",
      "Training time MLP: 50.72 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "epochs_EEG=mne.concatenate_epochs(data_EEG(subjects=subjects, multiclass=False))\n",
    "\n",
    "y = epochs_EEG.events[:, -1]\n",
    "X = csp.fit_transform(1e6 *epochs_EEG.get_data(), y)  \n",
    "y = le.fit_transform(y)\n",
    "\n",
    "X = torch.from_numpy(X).double()\n",
    "y  = torch.from_numpy(y).long()\n",
    "\n",
    "\n",
    "input_layer = X.shape[1]\n",
    "output_layer = len(np.unique(y))\n",
    "\n",
    "\n",
    "kan_model  = creating_models(input_layer, output_layer)\n",
    "kan_optimizer = LBFGS(kan_model.parameters(), lr=lr) \n",
    "\n",
    "mlp_model = MLP(input_layer, hidden_layers, output_layer).to(device)\n",
    "\n",
    "for param in mlp_model.parameters():\n",
    "    param.data = param.data.double()\n",
    "mlp_optimizer = LBFGS(mlp_model.parameters(), lr=lr, history_size=10, line_search_fn=\"strong_wolfe\", tolerance_grad=1e-32, tolerance_change=1e-32, tolerance_ys=1e-32) #(mlp_model.parameters(), lr=lr)\n",
    "\n",
    "\n",
    "metrics_KAN = {}  \n",
    "metrics_KAN['train'] = []\n",
    "metrics_KAN['test'] = []\n",
    "\n",
    "metrics_MLP = {}  \n",
    "metrics_MLP['train'] = []\n",
    "metrics_MLP['test'] = []\n",
    "\n",
    "total_train_time_KAN = 0\n",
    "total_train_time_MLP = 0\n",
    "\n",
    "for fold, (train_index, test_index) in enumerate(kf.split(X)):\n",
    "    # Prepare data for this fold\n",
    "    X_train, X_test = X[train_index], X[test_index]\n",
    "    y_train, y_test = y[train_index], y[test_index]\n",
    "\n",
    "    dataset={}\n",
    "    dataset['train_input']= X_train\n",
    "    dataset['train_label']= y_train\n",
    "    dataset['test_input']= X_test\n",
    "    dataset['test_label']= y_test\n",
    "\n",
    "    fold_metrics_KAN_train = []\n",
    "    fold_metrics_KAN_test =[]\n",
    "    fold_metrics_MLP_train = []\n",
    "    fold_metrics_MLP_test =[]\n",
    "\n",
    "    for epoch in range(epochs):\n",
    "        print('###### EPOCH ', epoch, \" ############\")\n",
    "\n",
    "        epoch_start_time_KAN = time.time()\n",
    "\n",
    "        results_kan = kan_model.train(dataset, opt=\"LBFGS\", steps=steps, \n",
    "        loss_fn=loss_fn, batch=-1, lr=lr)\n",
    "        \n",
    "        epoch_end_time_KAN = time.time()\n",
    "        epoch_train_KAN = epoch_end_time_KAN - epoch_start_time_KAN\n",
    "        total_train_time_KAN += epoch_train_KAN\n",
    "        \n",
    "        epoch_start_time_MLP = time.time()\n",
    "\n",
    "        fold_metrics_KAN_train.append(np.mean(results_kan['train_acc']))\n",
    "        fold_metrics_KAN_test.append(np.mean(results_kan['test_acc'])) \n",
    "\n",
    "        results_mlp = mlp_model.train(dataset, opt=\"LBFGS\", steps=steps, \n",
    "        loss_fn=loss_fn, batch=-1, lr=lr)\n",
    "\n",
    "        epoch_end_time_MLP = time.time()\n",
    "        epoch_train_MLP = epoch_end_time_MLP - epoch_start_time_MLP\n",
    "        total_train_time_MLP += epoch_train_MLP\n",
    "        \n",
    "        fold_metrics_MLP_train.append(np.mean(results_mlp['train_acc']))\n",
    "        fold_metrics_MLP_test.append(np.mean(results_mlp['test_acc'])) \n",
    "\n",
    "    metrics_KAN['train'].append(fold_metrics_KAN_train)\n",
    "    metrics_KAN['test'].append(fold_metrics_KAN_test)\n",
    "    \n",
    "    metrics_MLP['train'].append(fold_metrics_MLP_train)\n",
    "    metrics_MLP['test'].append(fold_metrics_MLP_test)\n",
    "\n",
    "    \n",
    "print(f\"Training time KAN: {total_train_time_KAN:.2f} seconds\")\n",
    "print(f\"Training time MLP: {total_train_time_MLP:.2f} seconds\")\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs_EEG=mne.concatenate_epochs(data_EEG(subjects=subjects, train=False))\n",
    "y = epochs_EEG.events[:, -1]\n",
    "X = csp.fit_transform(1e6 *epochs_EEG.get_data(), y)  \n",
    "y = le.fit_transform(y)\n",
    "X = torch.from_numpy(X).double()\n",
    " \n",
    "pred_start_time_KAN = time.time()\n",
    "output_KAN= kan_model(X)\n",
    "pred_time_KAN = time.time() - pred_start_time_KAN\n",
    "\n",
    "\n",
    "pred_start_time_MLP = time.time()\n",
    "output_MLP= mlp_model(X)\n",
    "pred_time_MLP = time.time() - pred_start_time_KAN\n",
    "\n",
    "metrics_KAN['pred']=torch.argmax(output_KAN, dim=1).numpy()\n",
    "metrics_KAN['real']=y\n",
    "metrics_KAN['time']=[total_train_time_KAN, pred_time_KAN]\n",
    "\n",
    "filename = 'two_class_CSP_KAN_'+str(subjects[0])+\".json\"\n",
    "export_json(metrics_KAN,filename)\n",
    "\n",
    "\n",
    "metrics_MLP['pred']=torch.argmax(output_MLP, dim=1).numpy()\n",
    "metrics_MLP['real']=y\n",
    "metrics_MLP['time']=[float(total_train_time_MLP), float(pred_time_MLP)]\n",
    "\n",
    "filename = 'two_class_CSP_MLP_'+str(subjects[0])+\".json\"\n",
    "export_json(metrics_MLP,filename)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>MULTICLASS CSP </h>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.644 | test acc: 0.500 | reg: 53.662 : 100%|████████████| 50/50 [00:05<00:00,  8.62it/s]\n",
      "train acc: 0.676 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 66.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.648 | test acc: 0.542 | reg: 75.457 : 100%|████████████| 50/50 [00:05<00:00,  8.71it/s]\n",
      "train acc: 0.671 | test acc: 0.375 : 100%|█████████████████████████| 50/50 [00:00<00:00, 126.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.657 | test acc: 0.542 | reg: 140.854 : 100%|███████████| 50/50 [00:06<00:00,  7.78it/s]\n",
      "train acc: 0.671 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 80.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.667 | test acc: 0.583 | reg: 167.807 : 100%|███████████| 50/50 [00:08<00:00,  5.85it/s]\n",
      "train acc: 0.671 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 86.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.667 | test acc: 0.583 | reg: 187.358 : 100%|███████████| 50/50 [00:08<00:00,  6.07it/s]\n",
      "train acc: 0.671 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 82.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.676 | test acc: 0.583 | reg: 249.058 : 100%|███████████| 50/50 [00:09<00:00,  5.19it/s]\n",
      "train acc: 0.671 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 82.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.671 | test acc: 0.625 | reg: 255.156 : 100%|███████████| 50/50 [00:09<00:00,  5.25it/s]\n",
      "train acc: 0.671 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 69.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.681 | test acc: 0.583 | reg: 317.193 : 100%|███████████| 50/50 [00:09<00:00,  5.27it/s]\n",
      "train acc: 0.671 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 87.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.685 | test acc: 0.625 | reg: 322.128 : 100%|███████████| 50/50 [00:07<00:00,  6.52it/s]\n",
      "train acc: 0.671 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 96.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.685 | test acc: 0.625 | reg: 335.042 : 100%|███████████| 50/50 [00:07<00:00,  6.51it/s]\n",
      "train acc: 0.671 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 96.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.681 | test acc: 0.625 | reg: 343.820 : 100%|███████████| 50/50 [00:07<00:00,  6.57it/s]\n",
      "train acc: 0.671 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 76.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.685 | test acc: 0.625 | reg: 351.589 : 100%|███████████| 50/50 [00:07<00:00,  6.27it/s]\n",
      "train acc: 0.671 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 78.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.694 | test acc: 0.583 | reg: 516.007 : 100%|███████████| 50/50 [00:07<00:00,  6.55it/s]\n",
      "train acc: 0.671 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 92.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.676 | test acc: 0.583 | reg: 520.896 : 100%|███████████| 50/50 [00:08<00:00,  6.22it/s]\n",
      "train acc: 0.671 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 88.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.690 | test acc: 0.583 | reg: 581.877 : 100%|███████████| 50/50 [00:08<00:00,  6.16it/s]\n",
      "train acc: 0.671 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 87.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.694 | test acc: 0.583 | reg: 580.138 : 100%|███████████| 50/50 [00:08<00:00,  6.24it/s]\n",
      "train acc: 0.671 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 91.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.690 | test acc: 0.583 | reg: 582.019 : 100%|███████████| 50/50 [00:08<00:00,  6.21it/s]\n",
      "train acc: 0.671 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 92.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.694 | test acc: 0.542 | reg: 601.570 : 100%|███████████| 50/50 [00:08<00:00,  5.98it/s]\n",
      "train acc: 0.671 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 88.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.699 | test acc: 0.583 | reg: 602.932 : 100%|███████████| 50/50 [00:08<00:00,  5.86it/s]\n",
      "train acc: 0.671 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 84.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.690 | test acc: 0.583 | reg: 619.275 : 100%|███████████| 50/50 [00:07<00:00,  6.29it/s]\n",
      "train acc: 0.671 | test acc: 0.375 : 100%|██████████████████████████| 50/50 [00:00<00:00, 87.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.667 | test acc: 0.792 | reg: 558.551 : 100%|███████████| 50/50 [00:08<00:00,  6.14it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|██████████████████████████| 50/50 [00:01<00:00, 36.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.648 | test acc: 0.750 | reg: 474.200 : 100%|███████████| 50/50 [00:08<00:00,  6.05it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 117.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.667 | test acc: 0.750 | reg: 453.785 : 100%|███████████| 50/50 [00:08<00:00,  5.95it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 107.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.667 | test acc: 0.750 | reg: 430.690 : 100%|███████████| 50/50 [00:09<00:00,  5.43it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 138.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.671 | test acc: 0.750 | reg: 414.824 : 100%|███████████| 50/50 [00:08<00:00,  5.77it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 137.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.662 | test acc: 0.750 | reg: 404.901 : 100%|███████████| 50/50 [00:09<00:00,  5.50it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 134.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.662 | test acc: 0.750 | reg: 389.748 : 100%|███████████| 50/50 [00:08<00:00,  6.18it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 123.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.667 | test acc: 0.750 | reg: 380.205 : 100%|███████████| 50/50 [00:09<00:00,  5.45it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 108.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.662 | test acc: 0.750 | reg: 372.147 : 100%|███████████| 50/50 [00:08<00:00,  5.59it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 139.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.662 | test acc: 0.750 | reg: 368.067 : 100%|███████████| 50/50 [00:09<00:00,  5.44it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 145.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.667 | test acc: 0.750 | reg: 369.577 : 100%|███████████| 50/50 [00:08<00:00,  5.69it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 143.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.662 | test acc: 0.708 | reg: 375.780 : 100%|███████████| 50/50 [00:09<00:00,  5.47it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 125.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.662 | test acc: 0.750 | reg: 377.710 : 100%|███████████| 50/50 [00:08<00:00,  5.59it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 149.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.662 | test acc: 0.708 | reg: 376.286 : 100%|███████████| 50/50 [00:09<00:00,  5.52it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 139.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.662 | test acc: 0.708 | reg: 377.395 : 100%|███████████| 50/50 [00:08<00:00,  5.74it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 125.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.667 | test acc: 0.708 | reg: 377.018 : 100%|███████████| 50/50 [00:08<00:00,  5.60it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 124.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.667 | test acc: 0.708 | reg: 377.969 : 100%|███████████| 50/50 [00:08<00:00,  5.67it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 120.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.667 | test acc: 0.708 | reg: 380.056 : 100%|███████████| 50/50 [00:08<00:00,  5.64it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 134.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.667 | test acc: 0.708 | reg: 380.450 : 100%|███████████| 50/50 [00:09<00:00,  5.37it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 143.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.662 | test acc: 0.708 | reg: 381.974 : 100%|███████████| 50/50 [00:08<00:00,  5.62it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 127.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.690 | test acc: 0.583 | reg: 398.246 : 100%|███████████| 50/50 [00:08<00:00,  6.10it/s]\n",
      "train acc: 0.667 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 113.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.694 | test acc: 0.583 | reg: 415.496 : 100%|███████████| 50/50 [00:08<00:00,  5.97it/s]\n",
      "train acc: 0.667 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 66.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.694 | test acc: 0.583 | reg: 446.228 : 100%|███████████| 50/50 [00:08<00:00,  5.71it/s]\n",
      "train acc: 0.676 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 82.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.699 | test acc: 0.583 | reg: 448.329 : 100%|███████████| 50/50 [00:07<00:00,  6.31it/s]\n",
      "train acc: 0.671 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 108.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.699 | test acc: 0.583 | reg: 453.576 : 100%|███████████| 50/50 [00:08<00:00,  6.10it/s]\n",
      "train acc: 0.671 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:01<00:00, 33.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.699 | test acc: 0.583 | reg: 455.432 : 100%|███████████| 50/50 [00:08<00:00,  6.06it/s]\n",
      "train acc: 0.671 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 138.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.699 | test acc: 0.583 | reg: 484.584 : 100%|███████████| 50/50 [00:08<00:00,  5.99it/s]\n",
      "train acc: 0.671 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 62.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.694 | test acc: 0.583 | reg: 484.678 : 100%|███████████| 50/50 [00:07<00:00,  6.27it/s]\n",
      "train acc: 0.671 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 121.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.699 | test acc: 0.583 | reg: 486.122 : 100%|███████████| 50/50 [00:07<00:00,  6.71it/s]\n",
      "train acc: 0.671 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 113.48it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.694 | test acc: 0.583 | reg: 487.980 : 100%|███████████| 50/50 [00:07<00:00,  6.66it/s]\n",
      "train acc: 0.671 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 94.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.699 | test acc: 0.583 | reg: 489.674 : 100%|███████████| 50/50 [00:07<00:00,  6.52it/s]\n",
      "train acc: 0.671 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 124.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.694 | test acc: 0.583 | reg: 490.630 : 100%|███████████| 50/50 [00:07<00:00,  6.71it/s]\n",
      "train acc: 0.671 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 133.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.699 | test acc: 0.583 | reg: 491.710 : 100%|███████████| 50/50 [00:07<00:00,  6.71it/s]\n",
      "train acc: 0.671 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 160.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.699 | test acc: 0.583 | reg: 494.271 : 100%|███████████| 50/50 [00:07<00:00,  6.51it/s]\n",
      "train acc: 0.671 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 159.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.694 | test acc: 0.583 | reg: 496.447 : 100%|███████████| 50/50 [00:07<00:00,  6.38it/s]\n",
      "train acc: 0.671 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 159.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.699 | test acc: 0.583 | reg: 497.545 : 100%|███████████| 50/50 [00:07<00:00,  6.43it/s]\n",
      "train acc: 0.671 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 144.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.699 | test acc: 0.583 | reg: 505.109 : 100%|███████████| 50/50 [00:07<00:00,  6.73it/s]\n",
      "train acc: 0.671 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 177.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.699 | test acc: 0.583 | reg: 505.406 : 100%|███████████| 50/50 [00:07<00:00,  6.63it/s]\n",
      "train acc: 0.671 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 177.78it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.694 | test acc: 0.583 | reg: 510.439 : 100%|███████████| 50/50 [00:07<00:00,  6.31it/s]\n",
      "train acc: 0.671 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 150.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.699 | test acc: 0.583 | reg: 510.830 : 100%|███████████| 50/50 [00:08<00:00,  6.21it/s]\n",
      "train acc: 0.671 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 137.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.676 | test acc: 0.667 | reg: 506.630 : 100%|███████████| 50/50 [00:08<00:00,  6.15it/s]\n",
      "train acc: 0.657 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 52.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.671 | test acc: 0.667 | reg: 505.222 : 100%|███████████| 50/50 [00:08<00:00,  5.89it/s]\n",
      "train acc: 0.657 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 154.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.676 | test acc: 0.667 | reg: 494.081 : 100%|███████████| 50/50 [00:08<00:00,  5.95it/s]\n",
      "train acc: 0.657 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 260.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.676 | test acc: 0.667 | reg: 490.286 : 100%|███████████| 50/50 [00:08<00:00,  6.04it/s]\n",
      "train acc: 0.657 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 92.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.671 | test acc: 0.667 | reg: 481.479 : 100%|███████████| 50/50 [00:07<00:00,  6.55it/s]\n",
      "train acc: 0.657 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 90.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.676 | test acc: 0.667 | reg: 481.948 : 100%|███████████| 50/50 [00:07<00:00,  6.41it/s]\n",
      "train acc: 0.657 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 74.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.671 | test acc: 0.667 | reg: 474.524 : 100%|███████████| 50/50 [00:08<00:00,  5.96it/s]\n",
      "train acc: 0.657 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 74.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.676 | test acc: 0.667 | reg: 475.230 : 100%|███████████| 50/50 [00:08<00:00,  5.87it/s]\n",
      "train acc: 0.657 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 97.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.667 | test acc: 0.667 | reg: 472.016 : 100%|███████████| 50/50 [00:07<00:00,  6.48it/s]\n",
      "train acc: 0.657 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 78.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.671 | test acc: 0.667 | reg: 471.783 : 100%|███████████| 50/50 [00:08<00:00,  5.59it/s]\n",
      "train acc: 0.657 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 109.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.676 | test acc: 0.667 | reg: 468.805 : 100%|███████████| 50/50 [00:08<00:00,  5.64it/s]\n",
      "train acc: 0.657 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 90.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.676 | test acc: 0.667 | reg: 469.497 : 100%|███████████| 50/50 [00:07<00:00,  6.42it/s]\n",
      "train acc: 0.657 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 89.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.676 | test acc: 0.667 | reg: 467.773 : 100%|███████████| 50/50 [00:08<00:00,  5.95it/s]\n",
      "train acc: 0.657 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 92.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.676 | test acc: 0.667 | reg: 468.577 : 100%|███████████| 50/50 [00:08<00:00,  6.06it/s]\n",
      "train acc: 0.657 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 74.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.676 | test acc: 0.667 | reg: 467.929 : 100%|███████████| 50/50 [00:08<00:00,  6.11it/s]\n",
      "train acc: 0.657 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 90.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.676 | test acc: 0.667 | reg: 468.636 : 100%|███████████| 50/50 [00:07<00:00,  6.38it/s]\n",
      "train acc: 0.657 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 83.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.676 | test acc: 0.667 | reg: 468.163 : 100%|███████████| 50/50 [00:08<00:00,  6.14it/s]\n",
      "train acc: 0.657 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 76.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.676 | test acc: 0.667 | reg: 468.800 : 100%|███████████| 50/50 [00:08<00:00,  6.07it/s]\n",
      "train acc: 0.657 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 74.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.676 | test acc: 0.667 | reg: 468.645 : 100%|███████████| 50/50 [00:07<00:00,  6.78it/s]\n",
      "train acc: 0.657 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 87.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.676 | test acc: 0.667 | reg: 469.776 : 100%|███████████| 50/50 [00:07<00:00,  6.41it/s]\n",
      "train acc: 0.657 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 89.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.657 | test acc: 0.667 | reg: 475.461 : 100%|███████████| 50/50 [00:07<00:00,  6.62it/s]\n",
      "train acc: 0.657 | test acc: 0.708 : 100%|██████████████████████████| 50/50 [00:00<00:00, 57.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.662 | test acc: 0.667 | reg: 466.414 : 100%|███████████| 50/50 [00:08<00:00,  6.23it/s]\n",
      "train acc: 0.657 | test acc: 0.708 : 100%|█████████████████████████| 50/50 [00:00<00:00, 128.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.662 | test acc: 0.667 | reg: 462.521 : 100%|███████████| 50/50 [00:08<00:00,  6.00it/s]\n",
      "train acc: 0.657 | test acc: 0.708 : 100%|█████████████████████████| 50/50 [00:00<00:00, 127.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.662 | test acc: 0.667 | reg: 465.458 : 100%|███████████| 50/50 [00:08<00:00,  6.05it/s]\n",
      "train acc: 0.657 | test acc: 0.708 : 100%|█████████████████████████| 50/50 [00:00<00:00, 160.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.662 | test acc: 0.667 | reg: 464.395 : 100%|███████████| 50/50 [00:08<00:00,  6.10it/s]\n",
      "train acc: 0.657 | test acc: 0.708 : 100%|█████████████████████████| 50/50 [00:00<00:00, 161.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.667 | test acc: 0.667 | reg: 466.720 : 100%|███████████| 50/50 [00:08<00:00,  6.07it/s]\n",
      "train acc: 0.657 | test acc: 0.708 : 100%|█████████████████████████| 50/50 [00:00<00:00, 146.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.667 | test acc: 0.667 | reg: 469.655 : 100%|███████████| 50/50 [00:07<00:00,  6.53it/s]\n",
      "train acc: 0.657 | test acc: 0.708 : 100%|█████████████████████████| 50/50 [00:00<00:00, 160.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.667 | test acc: 0.667 | reg: 472.883 : 100%|███████████| 50/50 [00:08<00:00,  5.81it/s]\n",
      "train acc: 0.657 | test acc: 0.708 : 100%|██████████████████████████| 50/50 [00:00<00:00, 99.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.662 | test acc: 0.667 | reg: 474.335 : 100%|███████████| 50/50 [00:09<00:00,  5.40it/s]\n",
      "train acc: 0.657 | test acc: 0.708 : 100%|█████████████████████████| 50/50 [00:00<00:00, 130.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.667 | test acc: 0.667 | reg: 482.992 : 100%|███████████| 50/50 [00:09<00:00,  5.44it/s]\n",
      "train acc: 0.657 | test acc: 0.708 : 100%|█████████████████████████| 50/50 [00:00<00:00, 156.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.667 | test acc: 0.667 | reg: 483.350 : 100%|███████████| 50/50 [00:07<00:00,  6.54it/s]\n",
      "train acc: 0.657 | test acc: 0.708 : 100%|█████████████████████████| 50/50 [00:00<00:00, 150.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.667 | test acc: 0.667 | reg: 491.578 : 100%|███████████| 50/50 [00:07<00:00,  6.55it/s]\n",
      "train acc: 0.657 | test acc: 0.708 : 100%|█████████████████████████| 50/50 [00:00<00:00, 155.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.667 | test acc: 0.667 | reg: 492.463 : 100%|███████████| 50/50 [00:07<00:00,  6.28it/s]\n",
      "train acc: 0.657 | test acc: 0.708 : 100%|█████████████████████████| 50/50 [00:00<00:00, 166.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.667 | test acc: 0.667 | reg: 493.364 : 100%|███████████| 50/50 [00:08<00:00,  6.25it/s]\n",
      "train acc: 0.657 | test acc: 0.708 : 100%|█████████████████████████| 50/50 [00:00<00:00, 168.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.671 | test acc: 0.667 | reg: 495.145 : 100%|███████████| 50/50 [00:07<00:00,  6.66it/s]\n",
      "train acc: 0.657 | test acc: 0.708 : 100%|█████████████████████████| 50/50 [00:00<00:00, 166.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.667 | test acc: 0.667 | reg: 496.370 : 100%|███████████| 50/50 [00:07<00:00,  6.89it/s]\n",
      "train acc: 0.657 | test acc: 0.708 : 100%|█████████████████████████| 50/50 [00:00<00:00, 181.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.667 | test acc: 0.667 | reg: 497.366 : 100%|███████████| 50/50 [00:07<00:00,  6.56it/s]\n",
      "train acc: 0.657 | test acc: 0.708 : 100%|█████████████████████████| 50/50 [00:00<00:00, 154.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.667 | test acc: 0.667 | reg: 498.182 : 100%|███████████| 50/50 [00:08<00:00,  6.19it/s]\n",
      "train acc: 0.657 | test acc: 0.708 : 100%|█████████████████████████| 50/50 [00:00<00:00, 176.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.667 | test acc: 0.667 | reg: 498.793 : 100%|███████████| 50/50 [00:07<00:00,  6.45it/s]\n",
      "train acc: 0.657 | test acc: 0.708 : 100%|█████████████████████████| 50/50 [00:00<00:00, 183.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.671 | test acc: 0.667 | reg: 499.904 : 100%|███████████| 50/50 [00:07<00:00,  6.52it/s]\n",
      "train acc: 0.657 | test acc: 0.708 : 100%|█████████████████████████| 50/50 [00:00<00:00, 183.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.713 | test acc: 0.542 | reg: 474.615 : 100%|███████████| 50/50 [00:07<00:00,  6.33it/s]\n",
      "train acc: 0.662 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 92.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.699 | test acc: 0.500 | reg: 470.019 : 100%|███████████| 50/50 [00:07<00:00,  6.42it/s]\n",
      "train acc: 0.662 | test acc: 0.625 : 100%|█████████████████████████| 50/50 [00:00<00:00, 110.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.713 | test acc: 0.500 | reg: 473.539 : 100%|███████████| 50/50 [00:07<00:00,  6.27it/s]\n",
      "train acc: 0.662 | test acc: 0.625 : 100%|█████████████████████████| 50/50 [00:00<00:00, 117.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.704 | test acc: 0.500 | reg: 481.163 : 100%|███████████| 50/50 [00:07<00:00,  6.58it/s]\n",
      "train acc: 0.662 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 98.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.704 | test acc: 0.500 | reg: 482.547 : 100%|███████████| 50/50 [00:07<00:00,  6.34it/s]\n",
      "train acc: 0.662 | test acc: 0.625 : 100%|█████████████████████████| 50/50 [00:00<00:00, 134.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.704 | test acc: 0.500 | reg: 487.031 : 100%|███████████| 50/50 [00:08<00:00,  6.15it/s]\n",
      "train acc: 0.662 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 107.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.704 | test acc: 0.500 | reg: 487.928 : 100%|███████████| 50/50 [00:08<00:00,  6.02it/s]\n",
      "train acc: 0.662 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 183.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.500 | reg: 492.547 : 100%|███████████| 50/50 [00:07<00:00,  6.59it/s]\n",
      "train acc: 0.662 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 121.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.500 | reg: 493.322 : 100%|███████████| 50/50 [00:07<00:00,  6.32it/s]\n",
      "train acc: 0.662 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 141.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.500 | reg: 496.882 : 100%|███████████| 50/50 [00:08<00:00,  6.14it/s]\n",
      "train acc: 0.662 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 177.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.500 | reg: 498.317 : 100%|███████████| 50/50 [00:07<00:00,  6.52it/s]\n",
      "train acc: 0.662 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 177.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.500 | reg: 501.569 : 100%|███████████| 50/50 [00:07<00:00,  6.51it/s]\n",
      "train acc: 0.662 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 145.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.500 | reg: 502.552 : 100%|███████████| 50/50 [00:08<00:00,  5.59it/s]\n",
      "train acc: 0.662 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 133.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.500 | reg: 507.495 : 100%|███████████| 50/50 [00:08<00:00,  5.57it/s]\n",
      "train acc: 0.662 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 129.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.500 | reg: 508.455 : 100%|███████████| 50/50 [00:08<00:00,  5.87it/s]\n",
      "train acc: 0.662 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 164.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.500 | reg: 508.521 : 100%|███████████| 50/50 [00:08<00:00,  5.66it/s]\n",
      "train acc: 0.662 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 142.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.500 | reg: 509.312 : 100%|███████████| 50/50 [00:08<00:00,  6.03it/s]\n",
      "train acc: 0.662 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 142.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.500 | reg: 510.015 : 100%|███████████| 50/50 [00:07<00:00,  6.37it/s]\n",
      "train acc: 0.662 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 172.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.500 | reg: 510.773 : 100%|███████████| 50/50 [00:08<00:00,  6.13it/s]\n",
      "train acc: 0.662 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 151.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.500 | reg: 513.137 : 100%|███████████| 50/50 [00:08<00:00,  6.20it/s]\n",
      "train acc: 0.662 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 146.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.718 | test acc: 0.458 | reg: 540.146 : 100%|███████████| 50/50 [00:08<00:00,  6.14it/s]\n",
      "train acc: 0.662 | test acc: 0.583 : 100%|██████████████████████████| 50/50 [00:01<00:00, 49.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.718 | test acc: 0.458 | reg: 547.853 : 100%|███████████| 50/50 [00:07<00:00,  6.26it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|██████████████████████████| 50/50 [00:00<00:00, 95.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.718 | test acc: 0.458 | reg: 553.882 : 100%|███████████| 50/50 [00:07<00:00,  6.38it/s]\n",
      "train acc: 0.662 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 138.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.458 | reg: 556.880 : 100%|███████████| 50/50 [00:07<00:00,  6.44it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 121.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.713 | test acc: 0.458 | reg: 560.891 : 100%|███████████| 50/50 [00:08<00:00,  6.18it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|██████████████████████████| 50/50 [00:00<00:00, 76.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.718 | test acc: 0.458 | reg: 562.496 : 100%|███████████| 50/50 [00:07<00:00,  7.00it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 127.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.458 | reg: 564.058 : 100%|███████████| 50/50 [00:07<00:00,  6.79it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 142.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.704 | test acc: 0.458 | reg: 570.585 : 100%|███████████| 50/50 [00:08<00:00,  6.15it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 131.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.458 | reg: 569.550 : 100%|███████████| 50/50 [00:08<00:00,  6.09it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 131.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.704 | test acc: 0.458 | reg: 569.791 : 100%|███████████| 50/50 [00:08<00:00,  5.95it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 110.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.713 | test acc: 0.458 | reg: 571.516 : 100%|███████████| 50/50 [00:07<00:00,  6.34it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 131.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.458 | reg: 572.080 : 100%|███████████| 50/50 [00:08<00:00,  6.13it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 110.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.713 | test acc: 0.458 | reg: 571.909 : 100%|███████████| 50/50 [00:08<00:00,  6.00it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 124.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.458 | reg: 571.732 : 100%|███████████| 50/50 [00:07<00:00,  6.51it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 160.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.713 | test acc: 0.458 | reg: 571.970 : 100%|███████████| 50/50 [00:07<00:00,  6.96it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 165.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.713 | test acc: 0.458 | reg: 572.170 : 100%|███████████| 50/50 [00:07<00:00,  6.77it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 131.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.713 | test acc: 0.458 | reg: 572.290 : 100%|███████████| 50/50 [00:07<00:00,  6.54it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 154.79it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.713 | test acc: 0.458 | reg: 572.359 : 100%|███████████| 50/50 [00:07<00:00,  6.54it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 166.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.458 | reg: 572.432 : 100%|███████████| 50/50 [00:07<00:00,  6.90it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 138.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.458 | reg: 573.480 : 100%|███████████| 50/50 [00:07<00:00,  6.77it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 134.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.417 | reg: 552.071 : 100%|███████████| 50/50 [00:07<00:00,  6.50it/s]\n",
      "train acc: 0.648 | test acc: 0.583 : 100%|██████████████████████████| 50/50 [00:00<00:00, 82.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.417 | reg: 545.740 : 100%|███████████| 50/50 [00:07<00:00,  6.65it/s]\n",
      "train acc: 0.662 | test acc: 0.583 : 100%|██████████████████████████| 50/50 [00:00<00:00, 90.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.417 | reg: 542.939 : 100%|███████████| 50/50 [00:07<00:00,  6.58it/s]\n",
      "train acc: 0.653 | test acc: 0.583 : 100%|██████████████████████████| 50/50 [00:00<00:00, 97.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.417 | reg: 540.984 : 100%|███████████| 50/50 [00:07<00:00,  6.62it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 101.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.417 | reg: 540.118 : 100%|███████████| 50/50 [00:08<00:00,  6.23it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 115.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.417 | reg: 540.788 : 100%|███████████| 50/50 [00:08<00:00,  6.08it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 113.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.417 | reg: 543.412 : 100%|███████████| 50/50 [00:08<00:00,  6.18it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|██████████████████████████| 50/50 [00:00<00:00, 81.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.417 | reg: 544.447 : 100%|███████████| 50/50 [00:08<00:00,  6.22it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 111.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.417 | reg: 545.605 : 100%|███████████| 50/50 [00:08<00:00,  6.24it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 195.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.417 | reg: 546.959 : 100%|███████████| 50/50 [00:07<00:00,  6.39it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 190.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.417 | reg: 548.637 : 100%|███████████| 50/50 [00:07<00:00,  6.64it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 164.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.417 | reg: 549.440 : 100%|███████████| 50/50 [00:08<00:00,  6.12it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 165.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.417 | reg: 551.550 : 100%|███████████| 50/50 [00:08<00:00,  6.01it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 133.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.417 | reg: 552.897 : 100%|███████████| 50/50 [00:07<00:00,  6.35it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 134.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.417 | reg: 553.600 : 100%|███████████| 50/50 [00:07<00:00,  6.41it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 176.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.417 | reg: 554.945 : 100%|███████████| 50/50 [00:09<00:00,  5.48it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 155.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.417 | reg: 555.611 : 100%|███████████| 50/50 [00:09<00:00,  5.21it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 170.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.417 | reg: 556.693 : 100%|███████████| 50/50 [00:07<00:00,  6.26it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 140.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.704 | test acc: 0.458 | reg: 590.432 : 100%|███████████| 50/50 [00:08<00:00,  5.87it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 172.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.704 | test acc: 0.458 | reg: 590.554 : 100%|███████████| 50/50 [00:07<00:00,  6.26it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 146.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.667 | reg: 575.242 : 100%|███████████| 50/50 [00:08<00:00,  5.92it/s]\n",
      "train acc: 0.667 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 84.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.699 | test acc: 0.667 | reg: 560.078 : 100%|███████████| 50/50 [00:07<00:00,  6.46it/s]\n",
      "train acc: 0.667 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 81.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.708 | test acc: 0.625 | reg: 542.662 : 100%|███████████| 50/50 [00:07<00:00,  6.76it/s]\n",
      "train acc: 0.667 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 101.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.704 | test acc: 0.625 | reg: 538.078 : 100%|███████████| 50/50 [00:08<00:00,  6.08it/s]\n",
      "train acc: 0.667 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 90.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.699 | test acc: 0.625 | reg: 527.878 : 100%|███████████| 50/50 [00:07<00:00,  6.72it/s]\n",
      "train acc: 0.667 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 105.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.699 | test acc: 0.625 | reg: 525.446 : 100%|███████████| 50/50 [00:07<00:00,  6.36it/s]\n",
      "train acc: 0.667 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 100.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.704 | test acc: 0.625 | reg: 523.450 : 100%|███████████| 50/50 [00:07<00:00,  6.59it/s]\n",
      "train acc: 0.667 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 80.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.704 | test acc: 0.625 | reg: 522.034 : 100%|███████████| 50/50 [00:07<00:00,  6.45it/s]\n",
      "train acc: 0.667 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 108.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.704 | test acc: 0.625 | reg: 521.548 : 100%|███████████| 50/50 [00:08<00:00,  6.21it/s]\n",
      "train acc: 0.667 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 116.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.699 | test acc: 0.625 | reg: 520.160 : 100%|███████████| 50/50 [00:07<00:00,  6.40it/s]\n",
      "train acc: 0.667 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 97.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.704 | test acc: 0.625 | reg: 520.348 : 100%|███████████| 50/50 [00:07<00:00,  6.37it/s]\n",
      "train acc: 0.667 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 88.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.699 | test acc: 0.625 | reg: 519.931 : 100%|███████████| 50/50 [00:08<00:00,  6.10it/s]\n",
      "train acc: 0.667 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 97.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.704 | test acc: 0.625 | reg: 520.027 : 100%|███████████| 50/50 [00:08<00:00,  6.23it/s]\n",
      "train acc: 0.667 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 85.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.704 | test acc: 0.625 | reg: 520.171 : 100%|███████████| 50/50 [00:08<00:00,  5.99it/s]\n",
      "train acc: 0.667 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 83.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.699 | test acc: 0.625 | reg: 520.008 : 100%|███████████| 50/50 [00:09<00:00,  5.36it/s]\n",
      "train acc: 0.667 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 102.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.704 | test acc: 0.625 | reg: 520.282 : 100%|███████████| 50/50 [00:08<00:00,  5.73it/s]\n",
      "train acc: 0.667 | test acc: 0.667 : 100%|██████████████████████████| 50/50 [00:00<00:00, 88.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.704 | test acc: 0.625 | reg: 520.528 : 100%|███████████| 50/50 [00:07<00:00,  6.68it/s]\n",
      "train acc: 0.667 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 110.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.704 | test acc: 0.625 | reg: 521.314 : 100%|███████████| 50/50 [00:07<00:00,  6.70it/s]\n",
      "train acc: 0.667 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 121.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.704 | test acc: 0.625 | reg: 521.358 : 100%|███████████| 50/50 [00:07<00:00,  6.49it/s]\n",
      "train acc: 0.667 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 103.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.699 | test acc: 0.625 | reg: 521.611 : 100%|███████████| 50/50 [00:07<00:00,  6.60it/s]\n",
      "train acc: 0.667 | test acc: 0.667 : 100%|█████████████████████████| 50/50 [00:00<00:00, 124.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  0  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.685 | test acc: 0.500 | reg: 545.132 : 100%|███████████| 50/50 [00:07<00:00,  6.52it/s]\n",
      "train acc: 0.657 | test acc: 0.625 : 100%|█████████████████████████| 50/50 [00:00<00:00, 112.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  1  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.694 | test acc: 0.458 | reg: 550.811 : 100%|███████████| 50/50 [00:07<00:00,  6.80it/s]\n",
      "train acc: 0.662 | test acc: 0.625 : 100%|██████████████████████████| 50/50 [00:00<00:00, 69.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  2  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.690 | test acc: 0.458 | reg: 559.604 : 100%|███████████| 50/50 [00:07<00:00,  6.79it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|██████████████████████████| 50/50 [00:00<00:00, 96.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  3  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.690 | test acc: 0.458 | reg: 555.958 : 100%|███████████| 50/50 [00:07<00:00,  6.67it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 150.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  4  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.690 | test acc: 0.458 | reg: 564.350 : 100%|███████████| 50/50 [00:07<00:00,  6.56it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|██████████████████████████| 50/50 [00:00<00:00, 97.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  5  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.690 | test acc: 0.458 | reg: 572.979 : 100%|███████████| 50/50 [00:07<00:00,  6.47it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 124.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  6  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.690 | test acc: 0.458 | reg: 574.984 : 100%|███████████| 50/50 [00:07<00:00,  6.58it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 198.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  7  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.690 | test acc: 0.458 | reg: 578.816 : 100%|███████████| 50/50 [00:07<00:00,  6.65it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 130.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  8  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.699 | test acc: 0.458 | reg: 629.399 : 100%|███████████| 50/50 [00:07<00:00,  6.41it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 160.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  9  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.694 | test acc: 0.458 | reg: 627.721 : 100%|███████████| 50/50 [00:07<00:00,  6.98it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 155.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  10  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.694 | test acc: 0.458 | reg: 655.823 : 100%|███████████| 50/50 [00:07<00:00,  6.64it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 141.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  11  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.694 | test acc: 0.458 | reg: 659.024 : 100%|███████████| 50/50 [00:08<00:00,  6.14it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 130.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  12  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.694 | test acc: 0.458 | reg: 668.057 : 100%|███████████| 50/50 [00:08<00:00,  6.12it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 131.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  13  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.694 | test acc: 0.458 | reg: 669.553 : 100%|███████████| 50/50 [00:08<00:00,  6.18it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 119.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  14  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.694 | test acc: 0.458 | reg: 671.844 : 100%|███████████| 50/50 [00:08<00:00,  6.20it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 131.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  15  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.694 | test acc: 0.458 | reg: 699.412 : 100%|███████████| 50/50 [00:08<00:00,  5.98it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 135.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  16  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.694 | test acc: 0.458 | reg: 700.998 : 100%|███████████| 50/50 [00:08<00:00,  6.24it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 127.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  17  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.694 | test acc: 0.458 | reg: 703.267 : 100%|███████████| 50/50 [00:07<00:00,  6.54it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 159.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  18  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.694 | test acc: 0.458 | reg: 707.882 : 100%|███████████| 50/50 [00:08<00:00,  5.86it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 118.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### EPOCH  19  ############\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train acc: 0.694 | test acc: 0.458 | reg: 710.103 : 100%|███████████| 50/50 [00:08<00:00,  5.59it/s]\n",
      "train acc: 0.657 | test acc: 0.583 : 100%|█████████████████████████| 50/50 [00:00<00:00, 118.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training time KAN: 1612.50 seconds\n",
      "Training time MLP: 90.91 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "epochs_EEG=mne.concatenate_epochs(data_EEG(subjects=subjects, multiclass=True))\n",
    "\n",
    "y = epochs_EEG.events[:, -1]\n",
    "X = csp.fit_transform(1e6 *epochs_EEG.get_data(), y)  \n",
    "y = le.fit_transform(y)\n",
    "\n",
    "X = torch.from_numpy(X).double()\n",
    "y  = torch.from_numpy(y).long()\n",
    "\n",
    "\n",
    "input_layer = X.shape[1]\n",
    "output_layer = len(np.unique(y))\n",
    "\n",
    "\n",
    "kan_model  = creating_models(input_layer, output_layer)\n",
    "kan_optimizer = LBFGS(kan_model.parameters(), lr=lr) \n",
    "\n",
    "mlp_model = MLP(input_layer, hidden_layers, output_layer).to(device)\n",
    "\n",
    "for param in mlp_model.parameters():\n",
    "    param.data = param.data.double()\n",
    "mlp_optimizer = LBFGS(mlp_model.parameters(), lr=lr, history_size=10, line_search_fn=\"strong_wolfe\", tolerance_grad=1e-32, tolerance_change=1e-32, tolerance_ys=1e-32) #(mlp_model.parameters(), lr=lr)\n",
    "\n",
    "\n",
    "metrics_KAN = {}  \n",
    "metrics_KAN['train'] = []\n",
    "metrics_KAN['test'] = []\n",
    "\n",
    "metrics_MLP = {}  \n",
    "metrics_MLP['train'] = []\n",
    "metrics_MLP['test'] = []\n",
    "\n",
    "total_train_time_KAN = 0\n",
    "total_train_time_MLP = 0\n",
    "\n",
    "for fold, (train_index, test_index) in enumerate(kf.split(X)):\n",
    "    # Prepare data for this fold\n",
    "    X_train, X_test = X[train_index], X[test_index]\n",
    "    y_train, y_test = y[train_index], y[test_index]\n",
    "\n",
    "    dataset={}\n",
    "    dataset['train_input']= X_train\n",
    "    dataset['train_label']= y_train\n",
    "    dataset['test_input']= X_test\n",
    "    dataset['test_label']= y_test\n",
    "\n",
    "    fold_metrics_KAN_train = []\n",
    "    fold_metrics_KAN_test =[]\n",
    "    fold_metrics_MLP_train = []\n",
    "    fold_metrics_MLP_test =[]\n",
    "\n",
    "    for epoch in range(epochs):\n",
    "        print('###### EPOCH ', epoch, \" ############\")\n",
    "\n",
    "        epoch_start_time_KAN = time.time()\n",
    "\n",
    "        results_kan = kan_model.train(dataset, opt=\"LBFGS\", steps=steps, \n",
    "        loss_fn=loss_fn, batch=-1, lr=lr)\n",
    "        \n",
    "        epoch_end_time_KAN = time.time()\n",
    "        epoch_train_KAN = epoch_end_time_KAN - epoch_start_time_KAN\n",
    "        total_train_time_KAN += epoch_train_KAN\n",
    "        \n",
    "        epoch_start_time_MLP = time.time()\n",
    "\n",
    "        fold_metrics_KAN_train.append(np.mean(results_kan['train_acc']))\n",
    "        fold_metrics_KAN_test.append(np.mean(results_kan['test_acc'])) \n",
    "\n",
    "        results_mlp = mlp_model.train(dataset, opt=\"LBFGS\", steps=steps, \n",
    "        loss_fn=loss_fn, batch=-1, lr=lr)\n",
    "\n",
    "        epoch_end_time_MLP = time.time()\n",
    "        epoch_train_MLP = epoch_end_time_MLP - epoch_start_time_MLP\n",
    "        total_train_time_MLP += epoch_train_MLP\n",
    "        \n",
    "        fold_metrics_MLP_train.append(np.mean(results_mlp['train_acc']))\n",
    "        fold_metrics_MLP_test.append(np.mean(results_mlp['test_acc'])) \n",
    "\n",
    "    metrics_KAN['train'].append(fold_metrics_KAN_train)\n",
    "    metrics_KAN['test'].append(fold_metrics_KAN_test)\n",
    "    \n",
    "    metrics_MLP['train'].append(fold_metrics_MLP_train)\n",
    "    metrics_MLP['test'].append(fold_metrics_MLP_test)\n",
    "\n",
    "    \n",
    "print(f\"Training time KAN: {total_train_time_KAN:.2f} seconds\")\n",
    "print(f\"Training time MLP: {total_train_time_MLP:.2f} seconds\")\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs_EEG=mne.concatenate_epochs(data_EEG(subjects=subjects, train=False, multiclass=True))\n",
    "y = epochs_EEG.events[:, -1]\n",
    "X = csp.fit_transform(1e6 *epochs_EEG.get_data(), y)  \n",
    "y = le.fit_transform(y)\n",
    "X = torch.from_numpy(X).double()\n",
    " \n",
    "pred_start_time_KAN = time.time()\n",
    "output_KAN= kan_model(X)\n",
    "pred_time_KAN = time.time() - pred_start_time_KAN\n",
    "\n",
    "\n",
    "pred_start_time_MLP = time.time()\n",
    "output_MLP= mlp_model(X)\n",
    "pred_time_MLP = time.time() - pred_start_time_KAN\n",
    "\n",
    "metrics_KAN['pred']=torch.argmax(output_KAN, dim=1).numpy()\n",
    "metrics_KAN['real']=y\n",
    "metrics_KAN['time']=[total_train_time_KAN, pred_time_KAN]\n",
    "\n",
    "filename = 'multiclass_CSP_KAN_'+str(subjects[0])+\".json\"\n",
    "export_json(metrics_KAN,filename)\n",
    "\n",
    "\n",
    "metrics_MLP['pred']=torch.argmax(output_MLP, dim=1).numpy()\n",
    "metrics_MLP['real']=y\n",
    "metrics_MLP['time']=[float(total_train_time_MLP), float(pred_time_MLP)]\n",
    "\n",
    "filename = 'multiclass_CSP_MLP_'+str(subjects[0])+\".json\"\n",
    "export_json(metrics_MLP,filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fixing (0,0,0) with tanh, r2=0.9987830927394489\n",
      "fixing (0,0,1) with tanh, r2=0.9987844456666457\n",
      "fixing (0,1,0) with tanh, r2=0.9974044397868516\n",
      "fixing (0,1,1) with tanh, r2=0.9978205290014331\n",
      "fixing (0,2,0) with tanh, r2=0.9988928062233559\n",
      "fixing (0,2,1) with tanh, r2=0.9994505995493326\n",
      "fixing (0,3,0) with sin, r2=0.9964715019308482\n",
      "fixing (0,3,1) with sin, r2=0.9964036477893259\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[420.97*sin(4.86*x_4 + 1.23) - 16122.29*tanh(2.21*x_1 + 3.39) + 780.98*tanh(4.54*x_2 + 5.6) + 2676.55*tanh(4.3*x_3 + 4.89) + 12335.46,\n",
       " -432.92*sin(4.87*x_4 + 1.24) + 15501.89*tanh(2.2*x_1 + 3.37) - 694.05*tanh(4.49*x_2 + 5.52) - 3972.74*tanh(3.2*x_3 + 3.64) - 10812.41]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lib = ['x','x^2','x^3','x^4','exp','log','sqrt','tanh','sin','abs']\n",
    "kan_model.auto_symbolic(lib=lib)\n",
    "formulas=kan_model.symbolic_formula()[0]\n",
    "\n",
    "formulas\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f'twoclass_CSP.txt', 'w') as f:  \n",
    "    for formula in formulas:\n",
    "        f.write(str(formula))\n",
    "        f.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sympy as sp\n",
    "\n",
    "def acc_multiclass(formulas, X, y):\n",
    "    batch = X.shape[0]\n",
    "    correct = 0\n",
    "    num_classes = len(formulas)\n",
    "    for i in range(batch):\n",
    "        logits = []\n",
    "        for formula in formulas:\n",
    "            logit = formula.subs({'x_1': X[i, 0], 'x_2': X[i, 1], 'x_3': X[i, 2],'x_4': X[i, 3]})\n",
    "            logit = sp.re(logit) if sp.im(logit) != 0 else logit  # Take the real part if complex\n",
    "            logits.append(float(logit.evalf())) \n",
    "        predicted_class = np.argmax(logits)\n",
    "        correct += (predicted_class == y[i])\n",
    "    return correct / batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_acc = acc_multiclass(formulas, dataset['train_input'], dataset['train_label'])\n",
    "test_acc = acc_multiclass(formulas, dataset['test_input'], dataset['test_label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.3750)"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.4537)"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pred=torch.argmax(mlp_model(X), dim=1)\n",
    "\n",
    "#sum(pred==y)/len(pred)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}