comparing_models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compare models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_metrics = torch.load(\"metrics/training\")\n",
    "val_metrics = torch.load(\"metrics/validation\")\n",
    "test_metrics = torch.load(\"metrics/test\")\n",
    "\n",
    "# Load all metrics into a dataframe\n",
    "df = pd.DataFrame.from_dict(train_metrics, orient='index').rename(columns={'accuracy':'train_acc', 'loss':'train_loss'})\n",
    "df.time = df.time / 360\n",
    "df.n_train_params = df.n_train_params / 1e4\n",
    "df.train_acc = df.train_acc.apply(max)\n",
    "df.train_loss = df.train_loss.apply(min)\n",
    "df = df.join(pd.DataFrame.from_dict(val_metrics, orient='index').drop('epochs', axis=1).rename(columns={'accuracy':'val_acc', 'loss':'val_loss'}))\n",
    "df.val_acc = df.val_acc.apply(max)\n",
    "df.val_loss = df.val_loss.apply(min)\n",
    "df = df.join(pd.DataFrame.from_dict(test_metrics, orient='index').rename(columns={'accuracy':'test_acc', 'loss':'test_loss'}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epochs</th>\n",
       "      <th>train_acc</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>time</th>\n",
       "      <th>n_train_params</th>\n",
       "      <th>val_acc</th>\n",
       "      <th>val_loss</th>\n",
       "      <th>test_acc</th>\n",
       "      <th>test_loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mobilenet</th>\n",
       "      <td>50</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000177</td>\n",
       "      <td>21.052294</td>\n",
       "      <td>16.7811</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000222</td>\n",
       "      <td>98.175247</td>\n",
       "      <td>0.081875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>densenet161</th>\n",
       "      <td>50</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000119</td>\n",
       "      <td>20.977644</td>\n",
       "      <td>28.9379</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000213</td>\n",
       "      <td>98.073872</td>\n",
       "      <td>0.078519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>densenet201</th>\n",
       "      <td>50</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000110</td>\n",
       "      <td>19.275755</td>\n",
       "      <td>25.1651</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000243</td>\n",
       "      <td>97.871121</td>\n",
       "      <td>0.086091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>resnext101</th>\n",
       "      <td>51</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000040</td>\n",
       "      <td>43.096268</td>\n",
       "      <td>26.8419</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000102</td>\n",
       "      <td>97.822638</td>\n",
       "      <td>0.091063</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>resnext50</th>\n",
       "      <td>52</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000053</td>\n",
       "      <td>28.438109</td>\n",
       "      <td>26.8419</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000145</td>\n",
       "      <td>97.782969</td>\n",
       "      <td>0.113843</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>densenet169</th>\n",
       "      <td>50</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000210</td>\n",
       "      <td>17.644150</td>\n",
       "      <td>21.8115</td>\n",
       "      <td>99.992614</td>\n",
       "      <td>0.001024</td>\n",
       "      <td>97.663963</td>\n",
       "      <td>0.106575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>densenet121</th>\n",
       "      <td>50</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000469</td>\n",
       "      <td>16.900645</td>\n",
       "      <td>13.4275</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000880</td>\n",
       "      <td>97.447990</td>\n",
       "      <td>0.105838</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shufflenet1.0</th>\n",
       "      <td>50</td>\n",
       "      <td>99.992614</td>\n",
       "      <td>0.054840</td>\n",
       "      <td>21.004696</td>\n",
       "      <td>13.4275</td>\n",
       "      <td>99.970456</td>\n",
       "      <td>0.058407</td>\n",
       "      <td>97.240832</td>\n",
       "      <td>0.199634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>resnet-wide50</th>\n",
       "      <td>50</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.001038</td>\n",
       "      <td>28.016335</td>\n",
       "      <td>26.8419</td>\n",
       "      <td>99.970456</td>\n",
       "      <td>0.002451</td>\n",
       "      <td>96.725141</td>\n",
       "      <td>0.147499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>resnet-wide101</th>\n",
       "      <td>13</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.002421</td>\n",
       "      <td>9.363903</td>\n",
       "      <td>26.8419</td>\n",
       "      <td>99.963070</td>\n",
       "      <td>0.003973</td>\n",
       "      <td>96.244711</td>\n",
       "      <td>0.154210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>inceptionv3</th>\n",
       "      <td>11</td>\n",
       "      <td>99.161635</td>\n",
       "      <td>0.033580</td>\n",
       "      <td>16.898179</td>\n",
       "      <td>36.9158</td>\n",
       "      <td>99.815348</td>\n",
       "      <td>0.008197</td>\n",
       "      <td>96.235896</td>\n",
       "      <td>0.141169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shufflenet0.5</th>\n",
       "      <td>50</td>\n",
       "      <td>99.748860</td>\n",
       "      <td>0.190823</td>\n",
       "      <td>19.191586</td>\n",
       "      <td>13.4275</td>\n",
       "      <td>99.719329</td>\n",
       "      <td>0.192792</td>\n",
       "      <td>95.376410</td>\n",
       "      <td>0.394630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>googlenet</th>\n",
       "      <td>50</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.001689</td>\n",
       "      <td>19.737618</td>\n",
       "      <td>13.4275</td>\n",
       "      <td>99.963070</td>\n",
       "      <td>0.002740</td>\n",
       "      <td>94.935649</td>\n",
       "      <td>0.211497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mnasnet1.0</th>\n",
       "      <td>6</td>\n",
       "      <td>99.998153</td>\n",
       "      <td>0.001457</td>\n",
       "      <td>3.081623</td>\n",
       "      <td>16.7811</td>\n",
       "      <td>32.919713</td>\n",
       "      <td>3.273800</td>\n",
       "      <td>29.716150</td>\n",
       "      <td>3.564803</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mnasnet0.5</th>\n",
       "      <td>7</td>\n",
       "      <td>98.301110</td>\n",
       "      <td>0.391318</td>\n",
       "      <td>2.299689</td>\n",
       "      <td>16.7811</td>\n",
       "      <td>21.870153</td>\n",
       "      <td>3.231128</td>\n",
       "      <td>21.033145</td>\n",
       "      <td>3.731022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>alexnet</th>\n",
       "      <td>23</td>\n",
       "      <td>98.849556</td>\n",
       "      <td>0.039134</td>\n",
       "      <td>8.161523</td>\n",
       "      <td>53.6707</td>\n",
       "      <td>99.896595</td>\n",
       "      <td>0.004215</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>resnet101</th>\n",
       "      <td>40</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000470</td>\n",
       "      <td>16.355889</td>\n",
       "      <td>26.8419</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000928</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>resnet152</th>\n",
       "      <td>40</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000457</td>\n",
       "      <td>18.678989</td>\n",
       "      <td>26.8419</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000846</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>resnet18</th>\n",
       "      <td>40</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.001234</td>\n",
       "      <td>12.354939</td>\n",
       "      <td>6.7203</td>\n",
       "      <td>99.992614</td>\n",
       "      <td>0.001778</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>resnet34</th>\n",
       "      <td>40</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.001317</td>\n",
       "      <td>12.899579</td>\n",
       "      <td>6.7203</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.001962</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>resnet50</th>\n",
       "      <td>40</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000607</td>\n",
       "      <td>14.185205</td>\n",
       "      <td>26.8419</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.001101</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>squeezenet1.0</th>\n",
       "      <td>40</td>\n",
       "      <td>98.469152</td>\n",
       "      <td>0.081213</td>\n",
       "      <td>12.574517</td>\n",
       "      <td>6.7203</td>\n",
       "      <td>98.544944</td>\n",
       "      <td>0.072962</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>squeezenet1.1</th>\n",
       "      <td>50</td>\n",
       "      <td>99.183794</td>\n",
       "      <td>0.052011</td>\n",
       "      <td>17.784571</td>\n",
       "      <td>6.7203</td>\n",
       "      <td>99.342640</td>\n",
       "      <td>0.037383</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg11</th>\n",
       "      <td>10</td>\n",
       "      <td>96.687164</td>\n",
       "      <td>0.159590</td>\n",
       "      <td>4.645867</td>\n",
       "      <td>53.6707</td>\n",
       "      <td>99.593766</td>\n",
       "      <td>0.011077</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg11_bn</th>\n",
       "      <td>16</td>\n",
       "      <td>99.951988</td>\n",
       "      <td>0.001902</td>\n",
       "      <td>7.683748</td>\n",
       "      <td>53.6707</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000155</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg13</th>\n",
       "      <td>11</td>\n",
       "      <td>96.511735</td>\n",
       "      <td>0.133864</td>\n",
       "      <td>5.659756</td>\n",
       "      <td>53.6707</td>\n",
       "      <td>99.571608</td>\n",
       "      <td>0.014835</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg13_bn</th>\n",
       "      <td>26</td>\n",
       "      <td>99.817185</td>\n",
       "      <td>0.012174</td>\n",
       "      <td>13.196006</td>\n",
       "      <td>53.6707</td>\n",
       "      <td>99.992614</td>\n",
       "      <td>0.000239</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16</th>\n",
       "      <td>12</td>\n",
       "      <td>94.979041</td>\n",
       "      <td>0.318009</td>\n",
       "      <td>6.413429</td>\n",
       "      <td>53.6707</td>\n",
       "      <td>98.640963</td>\n",
       "      <td>0.043861</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_bn</th>\n",
       "      <td>18</td>\n",
       "      <td>99.758093</td>\n",
       "      <td>0.007400</td>\n",
       "      <td>9.836070</td>\n",
       "      <td>53.6707</td>\n",
       "      <td>99.970456</td>\n",
       "      <td>0.001458</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg19</th>\n",
       "      <td>37</td>\n",
       "      <td>96.631766</td>\n",
       "      <td>0.157898</td>\n",
       "      <td>20.219687</td>\n",
       "      <td>53.6707</td>\n",
       "      <td>99.630697</td>\n",
       "      <td>0.012186</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg19_bn</th>\n",
       "      <td>9</td>\n",
       "      <td>99.628829</td>\n",
       "      <td>0.010607</td>\n",
       "      <td>5.117306</td>\n",
       "      <td>53.6707</td>\n",
       "      <td>99.940911</td>\n",
       "      <td>0.002471</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                epochs   train_acc  train_loss       time  n_train_params  \\\n",
       "mobilenet           50  100.000000    0.000177  21.052294         16.7811   \n",
       "densenet161         50  100.000000    0.000119  20.977644         28.9379   \n",
       "densenet201         50  100.000000    0.000110  19.275755         25.1651   \n",
       "resnext101          51  100.000000    0.000040  43.096268         26.8419   \n",
       "resnext50           52  100.000000    0.000053  28.438109         26.8419   \n",
       "densenet169         50  100.000000    0.000210  17.644150         21.8115   \n",
       "densenet121         50  100.000000    0.000469  16.900645         13.4275   \n",
       "shufflenet1.0       50   99.992614    0.054840  21.004696         13.4275   \n",
       "resnet-wide50       50  100.000000    0.001038  28.016335         26.8419   \n",
       "resnet-wide101      13  100.000000    0.002421   9.363903         26.8419   \n",
       "inceptionv3         11   99.161635    0.033580  16.898179         36.9158   \n",
       "shufflenet0.5       50   99.748860    0.190823  19.191586         13.4275   \n",
       "googlenet           50  100.000000    0.001689  19.737618         13.4275   \n",
       "mnasnet1.0           6   99.998153    0.001457   3.081623         16.7811   \n",
       "mnasnet0.5           7   98.301110    0.391318   2.299689         16.7811   \n",
       "alexnet             23   98.849556    0.039134   8.161523         53.6707   \n",
       "resnet101           40  100.000000    0.000470  16.355889         26.8419   \n",
       "resnet152           40  100.000000    0.000457  18.678989         26.8419   \n",
       "resnet18            40  100.000000    0.001234  12.354939          6.7203   \n",
       "resnet34            40  100.000000    0.001317  12.899579          6.7203   \n",
       "resnet50            40  100.000000    0.000607  14.185205         26.8419   \n",
       "squeezenet1.0       40   98.469152    0.081213  12.574517          6.7203   \n",
       "squeezenet1.1       50   99.183794    0.052011  17.784571          6.7203   \n",
       "vgg11               10   96.687164    0.159590   4.645867         53.6707   \n",
       "vgg11_bn            16   99.951988    0.001902   7.683748         53.6707   \n",
       "vgg13               11   96.511735    0.133864   5.659756         53.6707   \n",
       "vgg13_bn            26   99.817185    0.012174  13.196006         53.6707   \n",
       "vgg16               12   94.979041    0.318009   6.413429         53.6707   \n",
       "vgg16_bn            18   99.758093    0.007400   9.836070         53.6707   \n",
       "vgg19               37   96.631766    0.157898  20.219687         53.6707   \n",
       "vgg19_bn             9   99.628829    0.010607   5.117306         53.6707   \n",
       "\n",
       "                   val_acc  val_loss   test_acc  test_loss  \n",
       "mobilenet       100.000000  0.000222  98.175247   0.081875  \n",
       "densenet161     100.000000  0.000213  98.073872   0.078519  \n",
       "densenet201     100.000000  0.000243  97.871121   0.086091  \n",
       "resnext101      100.000000  0.000102  97.822638   0.091063  \n",
       "resnext50       100.000000  0.000145  97.782969   0.113843  \n",
       "densenet169      99.992614  0.001024  97.663963   0.106575  \n",
       "densenet121     100.000000  0.000880  97.447990   0.105838  \n",
       "shufflenet1.0    99.970456  0.058407  97.240832   0.199634  \n",
       "resnet-wide50    99.970456  0.002451  96.725141   0.147499  \n",
       "resnet-wide101   99.963070  0.003973  96.244711   0.154210  \n",
       "inceptionv3      99.815348  0.008197  96.235896   0.141169  \n",
       "shufflenet0.5    99.719329  0.192792  95.376410   0.394630  \n",
       "googlenet        99.963070  0.002740  94.935649   0.211497  \n",
       "mnasnet1.0       32.919713  3.273800  29.716150   3.564803  \n",
       "mnasnet0.5       21.870153  3.231128  21.033145   3.731022  \n",
       "alexnet          99.896595  0.004215        NaN        NaN  \n",
       "resnet101       100.000000  0.000928        NaN        NaN  \n",
       "resnet152       100.000000  0.000846        NaN        NaN  \n",
       "resnet18         99.992614  0.001778        NaN        NaN  \n",
       "resnet34        100.000000  0.001962        NaN        NaN  \n",
       "resnet50        100.000000  0.001101        NaN        NaN  \n",
       "squeezenet1.0    98.544944  0.072962        NaN        NaN  \n",
       "squeezenet1.1    99.342640  0.037383        NaN        NaN  \n",
       "vgg11            99.593766  0.011077        NaN        NaN  \n",
       "vgg11_bn        100.000000  0.000155        NaN        NaN  \n",
       "vgg13            99.571608  0.014835        NaN        NaN  \n",
       "vgg13_bn         99.992614  0.000239        NaN        NaN  \n",
       "vgg16            98.640963  0.043861        NaN        NaN  \n",
       "vgg16_bn         99.970456  0.001458        NaN        NaN  \n",
       "vgg19            99.630697  0.012186        NaN        NaN  \n",
       "vgg19_bn         99.940911  0.002471        NaN        NaN  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sort_values('test_acc', ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import required bokeh modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div class=\"bk-root\">\n",
       "        <a href=\"https://bokeh.org\" target=\"_blank\" class=\"bk-logo bk-logo-small bk-logo-notebook\"></a>\n",
       "        <span id=\"1001\">Loading BokehJS ...</span>\n",
       "    </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "(function(root) {\n",
       "  function now() {\n",
       "    return new Date();\n",
       "  }\n",
       "\n",
       "  var force = true;\n",
       "\n",
       "  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
       "    root._bokeh_onload_callbacks = [];\n",
       "    root._bokeh_is_loading = undefined;\n",
       "  }\n",
       "\n",
       "  var JS_MIME_TYPE = 'application/javascript';\n",
       "  var HTML_MIME_TYPE = 'text/html';\n",
       "  var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
       "  var CLASS_NAME = 'output_bokeh rendered_html';\n",
       "\n",
       "  /**\n",
       "   * Render data to the DOM node\n",
       "   */\n",
       "  function render(props, node) {\n",
       "    var script = document.createElement(\"script\");\n",
       "    node.appendChild(script);\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when an output is cleared or removed\n",
       "   */\n",
       "  function handleClearOutput(event, handle) {\n",
       "    var cell = handle.cell;\n",
       "\n",
       "    var id = cell.output_area._bokeh_element_id;\n",
       "    var server_id = cell.output_area._bokeh_server_id;\n",
       "    // Clean up Bokeh references\n",
       "    if (id != null && id in Bokeh.index) {\n",
       "      Bokeh.index[id].model.document.clear();\n",
       "      delete Bokeh.index[id];\n",
       "    }\n",
       "\n",
       "    if (server_id !== undefined) {\n",
       "      // Clean up Bokeh references\n",
       "      var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
       "      cell.notebook.kernel.execute(cmd, {\n",
       "        iopub: {\n",
       "          output: function(msg) {\n",
       "            var id = msg.content.text.trim();\n",
       "            if (id in Bokeh.index) {\n",
       "              Bokeh.index[id].model.document.clear();\n",
       "              delete Bokeh.index[id];\n",
       "            }\n",
       "          }\n",
       "        }\n",
       "      });\n",
       "      // Destroy server and session\n",
       "      var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
       "      cell.notebook.kernel.execute(cmd);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when a new output is added\n",
       "   */\n",
       "  function handleAddOutput(event, handle) {\n",
       "    var output_area = handle.output_area;\n",
       "    var output = handle.output;\n",
       "\n",
       "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
       "    if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
       "      return\n",
       "    }\n",
       "\n",
       "    var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
       "\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
       "      toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
       "      // store reference to embed id on output_area\n",
       "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
       "    }\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
       "      var bk_div = document.createElement(\"div\");\n",
       "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
       "      var script_attrs = bk_div.children[0].attributes;\n",
       "      for (var i = 0; i < script_attrs.length; i++) {\n",
       "        toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
       "        toinsert[toinsert.length - 1].firstChild.textContent = bk_div.children[0].textContent\n",
       "      }\n",
       "      // store reference to server id on output_area\n",
       "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
       "    }\n",
       "  }\n",
       "\n",
       "  function register_renderer(events, OutputArea) {\n",
       "\n",
       "    function append_mime(data, metadata, element) {\n",
       "      // create a DOM node to render to\n",
       "      var toinsert = this.create_output_subarea(\n",
       "        metadata,\n",
       "        CLASS_NAME,\n",
       "        EXEC_MIME_TYPE\n",
       "      );\n",
       "      this.keyboard_manager.register_events(toinsert);\n",
       "      // Render to node\n",
       "      var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
       "      render(props, toinsert[toinsert.length - 1]);\n",
       "      element.append(toinsert);\n",
       "      return toinsert\n",
       "    }\n",
       "\n",
       "    /* Handle when an output is cleared or removed */\n",
       "    events.on('clear_output.CodeCell', handleClearOutput);\n",
       "    events.on('delete.Cell', handleClearOutput);\n",
       "\n",
       "    /* Handle when a new output is added */\n",
       "    events.on('output_added.OutputArea', handleAddOutput);\n",
       "\n",
       "    /**\n",
       "     * Register the mime type and append_mime function with output_area\n",
       "     */\n",
       "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
       "      /* Is output safe? */\n",
       "      safe: true,\n",
       "      /* Index of renderer in `output_area.display_order` */\n",
       "      index: 0\n",
       "    });\n",
       "  }\n",
       "\n",
       "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
       "  if (root.Jupyter !== undefined) {\n",
       "    var events = require('base/js/events');\n",
       "    var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
       "\n",
       "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
       "      register_renderer(events, OutputArea);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  \n",
       "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
       "    root._bokeh_timeout = Date.now() + 5000;\n",
       "    root._bokeh_failed_load = false;\n",
       "  }\n",
       "\n",
       "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
       "     \"<div style='background-color: #fdd'>\\n\"+\n",
       "     \"<p>\\n\"+\n",
       "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
       "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
       "     \"</p>\\n\"+\n",
       "     \"<ul>\\n\"+\n",
       "     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
       "     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
       "     \"</ul>\\n\"+\n",
       "     \"<code>\\n\"+\n",
       "     \"from bokeh.resources import INLINE\\n\"+\n",
       "     \"output_notebook(resources=INLINE)\\n\"+\n",
       "     \"</code>\\n\"+\n",
       "     \"</div>\"}};\n",
       "\n",
       "  function display_loaded() {\n",
       "    var el = document.getElementById(\"1001\");\n",
       "    if (el != null) {\n",
       "      el.textContent = \"BokehJS is loading...\";\n",
       "    }\n",
       "    if (root.Bokeh !== undefined) {\n",
       "      if (el != null) {\n",
       "        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
       "      }\n",
       "    } else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(display_loaded, 100)\n",
       "    }\n",
       "  }\n",
       "\n",
       "\n",
       "  function run_callbacks() {\n",
       "    try {\n",
       "      root._bokeh_onload_callbacks.forEach(function(callback) {\n",
       "        if (callback != null)\n",
       "          callback();\n",
       "      });\n",
       "    } finally {\n",
       "      delete root._bokeh_onload_callbacks\n",
       "    }\n",
       "    console.debug(\"Bokeh: all callbacks have finished\");\n",
       "  }\n",
       "\n",
       "  function load_libs(css_urls, js_urls, callback) {\n",
       "    if (css_urls == null) css_urls = [];\n",
       "    if (js_urls == null) js_urls = [];\n",
       "\n",
       "    root._bokeh_onload_callbacks.push(callback);\n",
       "    if (root._bokeh_is_loading > 0) {\n",
       "      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
       "      return null;\n",
       "    }\n",
       "    if (js_urls == null || js_urls.length === 0) {\n",
       "      run_callbacks();\n",
       "      return null;\n",
       "    }\n",
       "    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
       "    root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
       "\n",
       "    function on_load() {\n",
       "      root._bokeh_is_loading--;\n",
       "      if (root._bokeh_is_loading === 0) {\n",
       "        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
       "        run_callbacks()\n",
       "      }\n",
       "    }\n",
       "\n",
       "    function on_error() {\n",
       "      console.error(\"failed to load \" + url);\n",
       "    }\n",
       "\n",
       "    for (var i = 0; i < css_urls.length; i++) {\n",
       "      var url = css_urls[i];\n",
       "      const element = document.createElement(\"link\");\n",
       "      element.onload = on_load;\n",
       "      element.onerror = on_error;\n",
       "      element.rel = \"stylesheet\";\n",
       "      element.type = \"text/css\";\n",
       "      element.href = url;\n",
       "      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
       "      document.body.appendChild(element);\n",
       "    }\n",
       "\n",
       "    const hashes = {\"https://cdn.bokeh.org/bokeh/release/bokeh-2.1.1.min.js\": \"kLr4fYcqcSpbuI95brIH3vnnYCquzzSxHPU6XGQCIkQRGJwhg0StNbj1eegrHs12\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.1.1.min.js\": \"xIGPmVtaOm+z0BqfSOMn4lOR6ciex448GIKG4eE61LsAvmGj48XcMQZtKcE/UXZe\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.1.1.min.js\": \"Dc9u1wF/0zApGIWoBbH77iWEHtdmkuYWG839Uzmv8y8yBLXebjO9ZnERsde5Ln/P\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-2.1.1.min.js\": \"cT9JaBz7GiRXdENrJLZNSC6eMNF3nh3fa5fTF51Svp+ukxPdwcU5kGXGPBgDCa2j\"};\n",
       "\n",
       "    for (var i = 0; i < js_urls.length; i++) {\n",
       "      var url = js_urls[i];\n",
       "      var element = document.createElement('script');\n",
       "      element.onload = on_load;\n",
       "      element.onerror = on_error;\n",
       "      element.async = false;\n",
       "      element.src = url;\n",
       "      if (url in hashes) {\n",
       "        element.crossOrigin = \"anonymous\";\n",
       "        element.integrity = \"sha384-\" + hashes[url];\n",
       "      }\n",
       "      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
       "      document.head.appendChild(element);\n",
       "    }\n",
       "  };\n",
       "\n",
       "  function inject_raw_css(css) {\n",
       "    const element = document.createElement(\"style\");\n",
       "    element.appendChild(document.createTextNode(css));\n",
       "    document.body.appendChild(element);\n",
       "  }\n",
       "\n",
       "  \n",
       "  var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-2.1.1.min.js\"];\n",
       "  var css_urls = [];\n",
       "  \n",
       "\n",
       "  var inline_js = [\n",
       "    function(Bokeh) {\n",
       "      Bokeh.set_log_level(\"info\");\n",
       "    },\n",
       "    function(Bokeh) {\n",
       "    \n",
       "    \n",
       "    }\n",
       "  ];\n",
       "\n",
       "  function run_inline_js() {\n",
       "    \n",
       "    if (root.Bokeh !== undefined || force === true) {\n",
       "      \n",
       "    for (var i = 0; i < inline_js.length; i++) {\n",
       "      inline_js[i].call(root, root.Bokeh);\n",
       "    }\n",
       "    if (force === true) {\n",
       "        display_loaded();\n",
       "      }} else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(run_inline_js, 100);\n",
       "    } else if (!root._bokeh_failed_load) {\n",
       "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
       "      root._bokeh_failed_load = true;\n",
       "    } else if (force !== true) {\n",
       "      var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n",
       "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
       "    }\n",
       "\n",
       "  }\n",
       "\n",
       "  if (root._bokeh_is_loading === 0) {\n",
       "    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "    run_inline_js();\n",
       "  } else {\n",
       "    load_libs(css_urls, js_urls, function() {\n",
       "      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
       "      run_inline_js();\n",
       "    });\n",
       "  }\n",
       "}(window));"
      ],
      "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n\n  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n    root._bokeh_onload_callbacks = [];\n    root._bokeh_is_loading = undefined;\n  }\n\n  \n\n  \n  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  var NB_LOAD_WARNING = {'data': {'text/html':\n     \"<div style='background-color: #fdd'>\\n\"+\n     \"<p>\\n\"+\n     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n     \"</p>\\n\"+\n     \"<ul>\\n\"+\n     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n     \"</ul>\\n\"+\n     \"<code>\\n\"+\n     \"from bokeh.resources import INLINE\\n\"+\n     \"output_notebook(resources=INLINE)\\n\"+\n     \"</code>\\n\"+\n     \"</div>\"}};\n\n  function display_loaded() {\n    var el = document.getElementById(\"1001\");\n    if (el != null) {\n      el.textContent = \"BokehJS is loading...\";\n    }\n    if (root.Bokeh !== undefined) {\n      if (el != null) {\n        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(display_loaded, 100)\n    }\n  }\n\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n\n    root._bokeh_onload_callbacks.push(callback);\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls == null || js_urls.length === 0) {\n      run_callbacks();\n      return null;\n    }\n    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }\n\n    const hashes = {\"https://cdn.bokeh.org/bokeh/release/bokeh-2.1.1.min.js\": \"kLr4fYcqcSpbuI95brIH3vnnYCquzzSxHPU6XGQCIkQRGJwhg0StNbj1eegrHs12\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.1.1.min.js\": \"xIGPmVtaOm+z0BqfSOMn4lOR6ciex448GIKG4eE61LsAvmGj48XcMQZtKcE/UXZe\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.1.1.min.js\": \"Dc9u1wF/0zApGIWoBbH77iWEHtdmkuYWG839Uzmv8y8yBLXebjO9ZnERsde5Ln/P\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-2.1.1.min.js\": \"cT9JaBz7GiRXdENrJLZNSC6eMNF3nh3fa5fTF51Svp+ukxPdwcU5kGXGPBgDCa2j\"};\n\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      if (url in hashes) {\n        element.crossOrigin = \"anonymous\";\n        element.integrity = \"sha384-\" + hashes[url];\n      }\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n  };\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  \n  var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-2.1.1.min.js\"];\n  var css_urls = [];\n  \n\n  var inline_js = [\n    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\n    function(Bokeh) {\n    \n    \n    }\n  ];\n\n  function run_inline_js() {\n    \n    if (root.Bokeh !== undefined || force === true) {\n      \n    for (var i = 0; i < inline_js.length; i++) {\n      inline_js[i].call(root, root.Bokeh);\n    }\n    if (force === true) {\n        display_loaded();\n      }} else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    } else if (force !== true) {\n      var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n    }\n\n  }\n\n  if (root._bokeh_is_loading === 0) {\n    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n    run_inline_js();\n  } else {\n    load_libs(css_urls, js_urls, function() {\n      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n      run_inline_js();\n    });\n  }\n}(window));"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from bokeh.plotting import figure as bfig, show as bshow\n",
    "from bokeh.models.tools import HoverTool\n",
    "from bokeh.models import CheckboxGroup, ColumnDataSource\n",
    "from bokeh.io import output_notebook\n",
    "from bokeh.layouts import gridplot, row\n",
    "output_notebook()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build interactive Bokeh plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.bokehjs_exec.v0+json": "",
      "text/html": [
       "\n",
       "<script id=\"4564\">\n",
       "  var xhr = new XMLHttpRequest()\n",
       "  xhr.responseType = 'blob';\n",
       "  xhr.open('GET', \"http://localhost:12863/autoload.js?bokeh-autoload-element=4564&bokeh-absolute-url=http://localhost:12863&resources=none\", true);\n",
       "  \n",
       "  xhr.onload = function (event) {\n",
       "    var script = document.createElement('script'),\n",
       "    src = URL.createObjectURL(event.target.response);\n",
       "    script.src = src;\n",
       "    document.body.appendChild(script);\n",
       "  };\n",
       "xhr.send();\n",
       "</script>"
      ]
     },
     "metadata": {
      "application/vnd.bokehjs_exec.v0+json": {
       "server_id": "93ccc74b95644c109e8b438a1312cd56"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def make_fig(src, x_col, y_col, sizes_col, title, size_mode, x_range, y_range, x_visible, y_visible, color=\"blue\", tooltips=[]):\n",
    "    f = bfig(title=title, sizing_mode=size_mode, x_range=x_range, y_range=y_range)\n",
    "    f.xaxis.visible = x_visible\n",
    "    f.yaxis.visible = y_visible\n",
    "    f.scatter(source=src, x=x_col, y=y_col, size=sizes_col, color=color)\n",
    "    tooltips = [('Model', '@index'),\n",
    "                ('Accuracy' if \"acc\" in y_col else 'Loss', '@%s' % y_col)] + tooltips\n",
    "    f.add_tools(HoverTool(tooltips=tooltips))\n",
    "    return f\n",
    "def bkapp(doc):\n",
    "    model_names = df.index.tolist()\n",
    "    active_models = [m for m in model_names if 'vgg' in m]\n",
    "    src = ColumnDataSource(data=df.loc[active_models, :])\n",
    "    size_mode = 'scale_width'\n",
    "    # # Accuracies\n",
    "    y_lim = (95, 100)\n",
    "    x_lim = (0, 52)\n",
    "    train_acc_fig = make_fig(src, \"epochs\", \"train_acc\", \"time\", \"Training Accuracy\", size_mode, x_lim, y_lim, \n",
    "                             False, True, tooltips=[('Training Time(hours)', '@time{0}/10')])\n",
    "    val_acc_fig = make_fig(src, \"epochs\", \"val_acc\", \"n_train_params\", \"Validation Accuracy\", size_mode, x_lim, train_acc_fig.y_range, \n",
    "                           False, False, color=\"orange\", tooltips=[('No of trainable parameters', '@n_train_params{0.0}e4')])\n",
    "    test_acc_fig = make_fig(src, \"epochs\", \"test_acc\", \"n_train_params\", \"Test Accuracy\", size_mode, x_lim, train_acc_fig.y_range, \n",
    "                            False, False, color=\"red\", tooltips=[('No of trainable parameters', '@n_train_params{0.0}e4')])\n",
    "    # # Losses\n",
    "    loss_y_lim = (0, 0.5)\n",
    "    train_loss_fig = make_fig(src, \"epochs\", \"train_loss\", \"time\", \"Training Loss\", size_mode, train_acc_fig.x_range, loss_y_lim, \n",
    "                              True, True, tooltips=[('Training Time(hours)', '@time{0}/10')])\n",
    "    val_loss_fig = make_fig(src, \"epochs\", \"val_loss\", \"n_train_params\", \"Validation Loss\", size_mode, val_acc_fig.x_range, train_loss_fig.y_range, \n",
    "                            True, False, color=\"orange\", tooltips=[('No of trainable parameters', '@n_train_params{0.0}e4')])\n",
    "    test_loss_fig = make_fig(src, \"epochs\", \"test_loss\", \"n_train_params\", \"Test Loss\", size_mode, test_acc_fig.x_range, train_loss_fig.y_range, \n",
    "                             True, False, color=\"red\", tooltips=[('No of trainable parameters', '@n_train_params{0.0}e4')])\n",
    "    def update(attr, old, new):\n",
    "        src.data.update(ColumnDataSource(df.loc[[model_names[i] for i in checkbox_group.active], :]).data)   \n",
    "    checkbox_group = CheckboxGroup(labels=model_names, active=[model_names.index(m) for m in active_models], width=200, height=700)\n",
    "    checkbox_group.on_change('active', update)\n",
    "    # Make grid\n",
    "    plots = row(checkbox_group, gridplot([[train_acc_fig, val_acc_fig, test_acc_fig],\n",
    "                                                [train_loss_fig, val_loss_fig, test_loss_fig]], sizing_mode='scale_both'))\n",
    "    doc.add_root(plots)\n",
    "bshow(bkapp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plot training and validation history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_metrics = torch.load(\"metrics/training\")\n",
    "val_metrics = torch.load(\"metrics/validation\")\n",
    "\n",
    "train_acc_df = pd.DataFrame.from_dict(train_metrics, orient=\"index\", columns=['epochs', 'accuracy']).accuracy.apply(lambda x: pd.Series(x)).T\n",
    "val_acc_df = pd.DataFrame.from_dict(train_metrics, orient=\"index\", columns=['epochs', 'accuracy']).accuracy.apply(lambda x: pd.Series(x)).T\n",
    "\n",
    "train_loss_df = pd.DataFrame.from_dict(train_metrics, orient=\"index\", columns=['epochs', 'loss']).loss.apply(lambda x: pd.Series(x)).T\n",
    "val_loss_df = pd.DataFrame.from_dict(train_metrics, orient=\"index\", columns=['epochs', 'loss']).loss.apply(lambda x: pd.Series(x)).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bokeh.palettes import Category20, Cividis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.bokehjs_exec.v0+json": "",
      "text/html": [
       "\n",
       "<script id=\"2158\">\n",
       "  var xhr = new XMLHttpRequest()\n",
       "  xhr.responseType = 'blob';\n",
       "  xhr.open('GET', \"http://localhost:34529/autoload.js?bokeh-autoload-element=2158&bokeh-absolute-url=http://localhost:34529&resources=none\", true);\n",
       "  \n",
       "  xhr.onload = function (event) {\n",
       "    var script = document.createElement('script'),\n",
       "    src = URL.createObjectURL(event.target.response);\n",
       "    script.src = src;\n",
       "    document.body.appendChild(script);\n",
       "  };\n",
       "xhr.send();\n",
       "</script>"
      ]
     },
     "metadata": {
      "application/vnd.bokehjs_exec.v0+json": {
       "server_id": "84d30b1dcd1b42b0a62751e39810e651"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def make_fig_history(title, size_mode, x_range, y_range, x_visible, y_visible):\n",
    "    f = bfig(title=title, sizing_mode=size_mode, x_range=x_range, y_range=y_range)\n",
    "    f.xaxis.visible = x_visible\n",
    "    f.yaxis.visible = y_visible\n",
    "    f.add_tools(HoverTool(tooltips=[('Model', '$name'),\n",
    "                                    ('Accuracy' if \"Acc\" in title else 'Loss', '$y')]))\n",
    "    return f\n",
    "def bkapp_history(doc):\n",
    "    models = dict((k, {'id': i, 'color': c}) for (i, k), c in zip(enumerate(train_acc_df.columns), Category20[20]+Cividis[11]))\n",
    "    active_models = [m for m in models if 'vgg' in m]\n",
    "    size_mode = 'scale_width'\n",
    "    # # Accuracies\n",
    "    y_lim = (95, 100)\n",
    "    x_lim = (0, 52)\n",
    "    train_acc_fig = make_fig_history(\"Training Accuracy\", size_mode, x_lim, y_lim, False, True)\n",
    "    val_acc_fig = make_fig_history(\"Validation Accuracy\", size_mode, x_lim, train_acc_fig.y_range, False, False)\n",
    "    # # Losses\n",
    "    loss_y_lim = (0, 0.5)\n",
    "    train_loss_fig = make_fig_history(\"Training Loss\", size_mode, train_acc_fig.x_range, loss_y_lim, True, True)\n",
    "    val_loss_fig = make_fig_history(\"Validation Loss\", size_mode, val_acc_fig.x_range, train_loss_fig.y_range, True, False)\n",
    "    \n",
    "    lines = {}\n",
    "    for model in models:\n",
    "        lines[model] = [\n",
    "            train_acc_fig.line(x='index', y=model, source=train_acc_df, color=models[model]['color'], name=model),\n",
    "            val_acc_fig.line(x='index', y=model, source=val_acc_df, color=models[model]['color'], name=model),\n",
    "            train_loss_fig.line(x='index', y=model, source=train_loss_df, color=models[model]['color'], name=model),\n",
    "            val_loss_fig.line(x='index', y=model, source=val_loss_df, color=models[model]['color'], name=model)\n",
    "        ]\n",
    "        if model not in active_models:\n",
    "            for i in range(4):\n",
    "                lines[model][i].visible = False\n",
    "    \n",
    "    def update(attr, old, new):\n",
    "        for name in models:\n",
    "            if models[name]['id'] in checkbox_group.active:\n",
    "                for i in range(4):\n",
    "                    lines[name][i].visible = True\n",
    "            else:\n",
    "                for i in range(4):\n",
    "                    lines[name][i].visible = False\n",
    "\n",
    "    checkbox_group = CheckboxGroup(labels=list(models.keys()), active=[models[m]['id'] for m in active_models], width=200, height=700)\n",
    "    checkbox_group.on_change('active', update)\n",
    "    # Make grid\n",
    "    plots = row(checkbox_group, gridplot([[train_acc_fig, val_acc_fig], [train_loss_fig, val_loss_fig]], sizing_mode='scale_both'))\n",
    "    doc.add_root(plots)\n",
    "bshow(bkapp_history)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}