train_meta_learner_prediction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "df = pd.read_pickle('T_combined_final_single_selector.pkl')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['fps_blend_comp'] = (np.around(np.vstack(df.fps_blend_comp),decimals=5 )).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "if gpus:\n",
    "  try:\n",
    "    # Currently, memory growth needs to be the same across GPUs\n",
    "    for gpu in gpus:\n",
    "      tf.config.experimental.set_memory_growth(gpu, True)\n",
    "    logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n",
    "    print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n",
    "  except RuntimeError as e:\n",
    "    # Memory growth must be set before GPUs have been initialized\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Removing the polymer blends with missing thermal property values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.dropna(subset = [\"value\"], inplace=True)\n",
    "df.reset_index(drop = True, inplace=True)\n",
    "# df.to_csv('T_dataset_processed_fox_fp.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.fillna(0, inplace=True)\n",
    "# np.array(df.bdid.values, dtype=str)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Changing the miscibility column to integer from boolean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['miscibility'] = df.miscibility.values.astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>selector</th>\n",
       "      <th>bdid</th>\n",
       "      <th>value</th>\n",
       "      <th>prop</th>\n",
       "      <th>type</th>\n",
       "      <th>c1</th>\n",
       "      <th>c2</th>\n",
       "      <th>miscibility</th>\n",
       "      <th>fps_blend_comp</th>\n",
       "      <th>val_1</th>\n",
       "      <th>val_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[0, 1, 0]</td>\n",
       "      <td>547</td>\n",
       "      <td>238.15</td>\n",
       "      <td>Tg</td>\n",
       "      <td>Blend</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.20</td>\n",
       "      <td>1</td>\n",
       "      <td>[0.20375, 0.85296, 0.12903, 0.27517, 0.25, 0.4...</td>\n",
       "      <td>238.15</td>\n",
       "      <td>238.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[0, 1, 0]</td>\n",
       "      <td>547</td>\n",
       "      <td>248.15</td>\n",
       "      <td>Tg</td>\n",
       "      <td>Blend</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.50</td>\n",
       "      <td>1</td>\n",
       "      <td>[0.26403, 0.66891, 0.07692, 0.19685, 0.14286, ...</td>\n",
       "      <td>248.15</td>\n",
       "      <td>248.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[0, 1, 0]</td>\n",
       "      <td>547</td>\n",
       "      <td>249.15</td>\n",
       "      <td>Tg</td>\n",
       "      <td>Blend</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.30</td>\n",
       "      <td>1</td>\n",
       "      <td>[0.2232, 0.78726, 0.11111, 0.24795, 0.21212, 0...</td>\n",
       "      <td>249.15</td>\n",
       "      <td>249.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[0, 1, 0]</td>\n",
       "      <td>547</td>\n",
       "      <td>338.15</td>\n",
       "      <td>Tg</td>\n",
       "      <td>Blend</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.50</td>\n",
       "      <td>1</td>\n",
       "      <td>[0.26403, 0.66891, 0.07692, 0.19685, 0.14286, ...</td>\n",
       "      <td>338.15</td>\n",
       "      <td>338.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[0, 1, 0]</td>\n",
       "      <td>547</td>\n",
       "      <td>342.15</td>\n",
       "      <td>Tg</td>\n",
       "      <td>Blend</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.20</td>\n",
       "      <td>1</td>\n",
       "      <td>[0.20375, 0.85296, 0.12903, 0.27517, 0.25, 0.4...</td>\n",
       "      <td>342.15</td>\n",
       "      <td>342.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23389</th>\n",
       "      <td>[0, 0, 2]</td>\n",
       "      <td>728</td>\n",
       "      <td>626.15</td>\n",
       "      <td>Tm</td>\n",
       "      <td>Blend</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.02493, 0...</td>\n",
       "      <td>601.15</td>\n",
       "      <td>626.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23390</th>\n",
       "      <td>[0, 0, 2]</td>\n",
       "      <td>728</td>\n",
       "      <td>627.15</td>\n",
       "      <td>Tm</td>\n",
       "      <td>Blend</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.07402, 0...</td>\n",
       "      <td>601.15</td>\n",
       "      <td>627.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23391</th>\n",
       "      <td>[0, 0, 2]</td>\n",
       "      <td>728</td>\n",
       "      <td>627.15</td>\n",
       "      <td>Tm</td>\n",
       "      <td>Blend</td>\n",
       "      <td>0.98</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.08632, 0...</td>\n",
       "      <td>601.15</td>\n",
       "      <td>627.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23392</th>\n",
       "      <td>[0, 0, 2]</td>\n",
       "      <td>728</td>\n",
       "      <td>627.15</td>\n",
       "      <td>Tm</td>\n",
       "      <td>Blend</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.08346, 0...</td>\n",
       "      <td>602.15</td>\n",
       "      <td>627.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23393</th>\n",
       "      <td>[0, 0, 2]</td>\n",
       "      <td>728</td>\n",
       "      <td>628.15</td>\n",
       "      <td>Tm</td>\n",
       "      <td>Blend</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.07872, 0...</td>\n",
       "      <td>601.15</td>\n",
       "      <td>628.15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>23394 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        selector bdid   value prop   type    c1    c2  miscibility  \\\n",
       "0      [0, 1, 0]  547  238.15   Tg  Blend  0.80  0.20            1   \n",
       "1      [0, 1, 0]  547  248.15   Tg  Blend  0.50  0.50            1   \n",
       "2      [0, 1, 0]  547  249.15   Tg  Blend  0.70  0.30            1   \n",
       "3      [0, 1, 0]  547  338.15   Tg  Blend  0.50  0.50            1   \n",
       "4      [0, 1, 0]  547  342.15   Tg  Blend  0.80  0.20            1   \n",
       "...          ...  ...     ...  ...    ...   ...   ...          ...   \n",
       "23389  [0, 0, 2]  728  626.15   Tm  Blend  0.30  0.70            0   \n",
       "23390  [0, 0, 2]  728  627.15   Tm  Blend  0.85  0.15            0   \n",
       "23391  [0, 0, 2]  728  627.15   Tm  Blend  0.98  0.02            0   \n",
       "23392  [0, 0, 2]  728  627.15   Tm  Blend  0.95  0.05            0   \n",
       "23393  [0, 0, 2]  728  628.15   Tm  Blend  0.90  0.10            0   \n",
       "\n",
       "                                          fps_blend_comp   val_1   val_2  \n",
       "0      [0.20375, 0.85296, 0.12903, 0.27517, 0.25, 0.4...  238.15  238.15  \n",
       "1      [0.26403, 0.66891, 0.07692, 0.19685, 0.14286, ...  248.15  248.15  \n",
       "2      [0.2232, 0.78726, 0.11111, 0.24795, 0.21212, 0...  249.15  249.15  \n",
       "3      [0.26403, 0.66891, 0.07692, 0.19685, 0.14286, ...  338.15  338.15  \n",
       "4      [0.20375, 0.85296, 0.12903, 0.27517, 0.25, 0.4...  342.15  342.15  \n",
       "...                                                  ...     ...     ...  \n",
       "23389  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.02493, 0...  601.15  626.15  \n",
       "23390  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.07402, 0...  601.15  627.15  \n",
       "23391  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.08632, 0...  601.15  627.15  \n",
       "23392  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.08346, 0...  602.15  627.15  \n",
       "23393  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.07872, 0...  601.15  628.15  \n",
       "\n",
       "[23394 rows x 11 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['composition_ratio_of_1st_coplymer'] = [[100,0,0]]*df.shape[0]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preparing the dataset for training using tensorflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "# os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
    "\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import QuantileTransformer, MinMaxScaler\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold\n",
    "# import pandas as pd\n",
    "# import tensorflow as tf\n",
    "tf.random.set_seed(123)\n",
    "\n",
    "\n",
    "df = df.sample(frac=1, random_state=123)\n",
    "\n",
    "target_property_scaler = MinMaxScaler\n",
    "\n",
    "batch_size = 200\n",
    "\n",
    "df['scaled_value'] = df['value']\n",
    "skf = StratifiedKFold(n_splits=5)\n",
    "\n",
    "props = df.prop.unique().tolist()\n",
    "\n",
    "datasets = []\n",
    "def get_dataset(df):\n",
    "    fps_blends = np.stack(df.fps_blend_comp.values).astype(np.float32)\n",
    "\n",
    "    selector = np.stack(df.selector.values).astype(np.float32)\n",
    "    cp_c1 = np.stack(df.composition_ratio_of_1st_coplymer.values).astype(np.float32)\n",
    "\n",
    "    target = df.scaled_value.values.astype(np.float32)[:, np.newaxis]\n",
    "    bdid = np.stack(df.bdid.values).astype(np.str)[:, np.newaxis]\n",
    "    \n",
    "    dataset = tf.data.Dataset.from_tensor_slices(({'selector': selector, 'fps_blends': fps_blends, 'prop': df.prop, 'c1': df.c1[..., None], 'cp_c1':cp_c1, 'miscibility':df.miscibility[..., None] ,'type': df.type[..., None], 'bdid':bdid}, target))\n",
    "\n",
    "    dataset = dataset.cache().batch(batch_size).prefetch(tf.data.experimental.AUTOTUNE)\n",
    "    return dataset\n",
    "\n",
    "\n",
    "training_df, test_df = train_test_split(df, test_size=0.2, stratify=df.prop, random_state=123)\n",
    "training_df, test_df = training_df.copy(), test_df.copy()\n",
    "\n",
    "for train_index, val_index in skf.split(training_df, training_df.prop):\n",
    "    \n",
    "    train_df = df.iloc[train_index].copy()\n",
    "    val_df = df.iloc[val_index].copy()\n",
    "    \n",
    "    # scale target values\n",
    "    property_scaler = {}\n",
    "    for prop in props:\n",
    "        property_scaler[prop] = target_property_scaler()\n",
    "\n",
    "        # train\n",
    "        cond = train_df[train_df.prop == prop].index\n",
    "        val = train_df.loc[cond, ['scaled_value']]\n",
    "        train_df.loc[cond, ['scaled_value']] = property_scaler[prop].fit_transform(val)\n",
    "\n",
    "        # val\n",
    "        cond = val_df[val_df.prop == prop].index\n",
    "        val = val_df.loc[cond, ['scaled_value']]\n",
    "        val_df.loc[cond, ['scaled_value']] = property_scaler[prop].transform(val)\n",
    "\n",
    "    datasets.append({'train': get_dataset(train_df), 'val': get_dataset(val_df), 'property_scaler': property_scaler})\n",
    "\n",
    "# Create final dataset for ensemble\n",
    "property_scaler_final = {}\n",
    "\n",
    "for prop in props:\n",
    "    property_scaler_final[prop] = target_property_scaler()\n",
    "    \n",
    "    # train\n",
    "    cond = training_df[training_df.prop == prop].index\n",
    "    val = training_df.loc[cond, ['scaled_value']]\n",
    "    training_df.loc[cond, ['scaled_value']] = property_scaler_final[prop].fit_transform(val)\n",
    "\n",
    "    # val\n",
    "    cond = test_df[test_df.prop == prop].index\n",
    "    val = test_df.loc[cond, ['scaled_value']]\n",
    "    test_df.loc[cond, ['scaled_value']] = property_scaler_final[prop].transform(val)\n",
    "    \n",
    "datasets_final = {'train': get_dataset(training_df), 'test': get_dataset(test_df), 'property_scaler': property_scaler_final}\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'train': <PrefetchDataset element_spec=({'selector': TensorSpec(shape=(None, 3), dtype=tf.float32, name=None), 'fps_blends': TensorSpec(shape=(None, 872), dtype=tf.float32, name=None), 'prop': TensorSpec(shape=(None,), dtype=tf.string, name=None), 'c1': TensorSpec(shape=(None, 1), dtype=tf.float64, name=None), 'cp_c1': TensorSpec(shape=(None, 3), dtype=tf.float32, name=None), 'miscibility': TensorSpec(shape=(None, 1), dtype=tf.int64, name=None), 'type': TensorSpec(shape=(None, 1), dtype=tf.string, name=None), 'bdid': TensorSpec(shape=(None, 1), dtype=tf.string, name=None)}, TensorSpec(shape=(None, 1), dtype=tf.float32, name=None))>,\n",
       " 'test': <PrefetchDataset element_spec=({'selector': TensorSpec(shape=(None, 3), dtype=tf.float32, name=None), 'fps_blends': TensorSpec(shape=(None, 872), dtype=tf.float32, name=None), 'prop': TensorSpec(shape=(None,), dtype=tf.string, name=None), 'c1': TensorSpec(shape=(None, 1), dtype=tf.float64, name=None), 'cp_c1': TensorSpec(shape=(None, 3), dtype=tf.float32, name=None), 'miscibility': TensorSpec(shape=(None, 1), dtype=tf.int64, name=None), 'type': TensorSpec(shape=(None, 1), dtype=tf.string, name=None), 'bdid': TensorSpec(shape=(None, 1), dtype=tf.string, name=None)}, TensorSpec(shape=(None, 1), dtype=tf.float32, name=None))>,\n",
       " 'property_scaler': {'Tm': MinMaxScaler(),\n",
       "  'Td': MinMaxScaler(),\n",
       "  'Tg': MinMaxScaler()}}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datasets_final"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Defining the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import tensorflow.keras as tfk\n",
    "import tensorflow as tf\n",
    "from datetime import datetime \n",
    "from tensorflow.python.keras.engine import data_adapter\n",
    "from keras_tuner import HyperModel\n",
    "from keras_tuner.tuners import Hyperband, RandomSearch\n",
    "import tensorflow_addons as tfa\n",
    "\n",
    "\n",
    "df.fps_blend_comp.to_pickle('pkl/fingerprints_prediction_final.pkl')\n",
    "class MetaModel(tfk.Model):\n",
    "    def __init__(self, hp): #, scale):\n",
    "        super().__init__()\n",
    "        self.base_models = []\n",
    "        # self.scaler_type = scale\n",
    "        for num in range(5):\n",
    "            model = tf.keras.models.load_model(f'models/cv_final_model/{num}')\n",
    "            model.trainable = False\n",
    "            self.base_models.append(model)\n",
    "        \n",
    "        self.my_layers = []\n",
    "        \n",
    "        for i in range(hp.Int('num_layers', 1, 2)): \n",
    "            new_step = [               \n",
    "            tf.keras.layers.Dense(units=hp.Int('units_' + str(i),\n",
    "                                            min_value=64,\n",
    "                                            max_value=544,\n",
    "                                            step=64),),\n",
    "            \n",
    "            tf.keras.layers.PReLU(),\n",
    "            tf.keras.layers.Dropout(hp.Float(\n",
    "                'dropout_' + str(i),\n",
    "                min_value=0.0,\n",
    "                max_value=0.5,\n",
    "                default=0.25,\n",
    "                step=0.05,\n",
    "            )),]\n",
    "            self.my_layers.append(new_step)\n",
    "            \n",
    "        self.my_layers.append([tf.keras.layers.Dense(1)])\n",
    "\n",
    "    def call(self, inputs, training=None): \n",
    "        \n",
    "        # drop prop\n",
    "        if 'prop' in inputs:\n",
    "            del inputs['prop']\n",
    "        x = []\n",
    "        for base in self.base_models:\n",
    "            if training:\n",
    "                res = base.call(inputs, training)\n",
    "            else:\n",
    "                res = base.call(inputs)\n",
    "            x.append(res)\n",
    "        x = tf.concat(x, -1)\n",
    "        \n",
    "        for num, layer_step in enumerate(self.my_layers):\n",
    "            for layer in layer_step:\n",
    "                x = layer(x)\n",
    "\n",
    "        return x\n",
    "    \n",
    "    def predict_step(self, data):\n",
    "        data = data_adapter.expand_1d(data)\n",
    "        x, _, _ = data_adapter.unpack_x_y_sample_weight(data)\n",
    "        # drop prop here\n",
    "        prop = x['prop']\n",
    "        # bdid = x['bdid']\n",
    "        type_poly = x['type']\n",
    "        selector = x['selector']\n",
    "        bdid = x['bdid']\n",
    "        c1 = x['c1']\n",
    "        cp_c1 = x['cp_c1']\n",
    "        miscibility = x['miscibility']\n",
    "        del x['prop']\n",
    "        # del x['bdid']\n",
    "        # del x['selector']\n",
    "        return self(x, training=True), data[-1], prop, type_poly, bdid, miscibility, c1, cp_c1, selector\n",
    "\n",
    "    \n",
    "    @tf.function(input_signature=[])\n",
    "    def fp_order(self):\n",
    "        fp_order = tf.convert_to_tensor(list(pd.read_pickle('pkl/fingerprints_prediction_final.pkl').columns))\n",
    "        return fp_order\n",
    "        \n",
    "    \n",
    "def build_model(hp):\n",
    "    model = MetaModel(hp)\n",
    "    opt = tf.keras.optimizers.Adam(\n",
    "            hp.Choice('learning_rate',\n",
    "                      values=[1e-3]))\n",
    "    opt = tfa.optimizers.SWA(opt)\n",
    "\n",
    "    model.compile(\n",
    "        optimizer=opt,\n",
    "        loss='mse',)\n",
    "    return model"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training and optimizing the hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import IPython\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "results, property_metric = [], []\n",
    "\n",
    "\n",
    "tuner = Hyperband(\n",
    "    build_model,\n",
    "    objective='val_loss',\n",
    "    max_epochs=300,\n",
    "    # hyperband_iterations=3,\n",
    "    seed=10,\n",
    "    directory=f'hyperparameter_search_meta_learner_final',\n",
    "    project_name='fold_0',\n",
    "    )\n",
    "\n",
    "reduce_lr = tfk.callbacks.ReduceLROnPlateau(\n",
    "    factor=0.9,\n",
    "    monitor=\"val_loss\",\n",
    "    verbose=1,\n",
    ")\n",
    "\n",
    "earlystop = tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=40)\n",
    "\n",
    "class ClearTrainingOutput(tf.keras.callbacks.Callback):\n",
    "    def on_train_end(*args, **kwargs):\n",
    "        IPython.display.clear_output(wait = True)\n",
    "\n",
    "# Create an instance of the model\n",
    "tuner.search(datasets_final['test'],\n",
    "            epochs=300,\n",
    "            validation_data=datasets_final['train'],\n",
    "            callbacks=[earlystop, reduce_lr, ClearTrainingOutput()],\n",
    "            verbose=0\n",
    "            )\n",
    "\n",
    "# Post processing\n",
    "best_values = tuner.get_best_hyperparameters()[0].values\n",
    "\n",
    "best_model = tuner.get_best_models(1)[0]\n",
    "res = np.concatenate(best_model.predict(datasets_final['train']), -1)\n",
    "# best_model.save(f'models/meta_model_{what}/', include_optimizer=False)\n",
    "\n",
    "\n",
    "_df = pd.DataFrame(res, columns=['pred', 'target', 'prop', 'type_poly', 'bdid', 'miscibility','c1', 'cp_c1_0', 'cp_c1_1', 'cp_c1_3', 'selector_0', 'selector_1', 'selector_2'])\n",
    "_df['prop'] = _df.prop.apply(lambda x: x.decode('utf-8'))\n",
    "# _df['bdid'] = _df.prop.apply(lambda x: x.decode('utf-8'))\n",
    "\n",
    "props = _df.prop.unique()\n",
    "\n",
    "property_scaler = datasets_final['property_scaler']\n",
    "for prop in props:\n",
    "\n",
    "    cond = _df[_df.prop == prop].index\n",
    "    rmse_scaled = mean_squared_error(_df.loc[cond, ['target']], _df.loc[cond, ['pred']], squared=False)\n",
    "    r2_scaled = r2_score(_df.loc[cond, ['target']], _df.loc[cond, ['pred']])\n",
    "    \n",
    "    _df.loc[cond, ['pred']] = property_scaler[prop].inverse_transform(_df.loc[cond, ['pred']])\n",
    "    _df.loc[cond, ['target']] = property_scaler[prop].inverse_transform(_df.loc[cond, ['target']])\n",
    "    \n",
    "    rmse = mean_squared_error(_df.loc[cond, ['target']], _df.loc[cond, ['pred']], squared=False)\n",
    "    r2 = r2_score(_df.loc[cond, ['target']], _df.loc[cond, ['pred']])\n",
    "    property_metric.append({'name': f'meta_learner', 'prop': prop, 'rmse': rmse, 'r2':r2, 'rmse_scaled': rmse_scaled, 'r2_scaled': r2_scaled})\n",
    "    \n",
    "# No scaling back\n",
    "rmse = mean_squared_error(res[:,0], res[:,1], squared=False)\n",
    "r2 = r2_score(res[:,0], res[:,1])\n",
    "\n",
    "results.append({'name': f'meta_learner', 'r2': r2, 'rmse':rmse})\n",
    "\n",
    "pd.DataFrame(results)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'num_layers': 2,\n",
       " 'units_0': 320,\n",
       " 'dropout_0': 0.0,\n",
       " 'learning_rate': 0.001,\n",
       " 'units_1': 384,\n",
       " 'dropout_1': 0.25,\n",
       " 'tuner/epochs': 100,\n",
       " 'tuner/initial_epoch': 34,\n",
       " 'tuner/bracket': 4,\n",
       " 'tuner/round': 3,\n",
       " 'tuner/trial_id': 'fa11f84b740e4ec853b33c7756e77695'}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tg</th>\n",
       "      <th>Td</th>\n",
       "      <th>Tm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>rmse</th>\n",
       "      <td>26.271093</td>\n",
       "      <td>44.638685</td>\n",
       "      <td>23.287833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mae</th>\n",
       "      <td>18.618491</td>\n",
       "      <td>31.086444</td>\n",
       "      <td>16.986242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>r2</th>\n",
       "      <td>0.862456</td>\n",
       "      <td>0.822291</td>\n",
       "      <td>0.891076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rmse_scaled</th>\n",
       "      <td>0.033129</td>\n",
       "      <td>0.050602</td>\n",
       "      <td>0.036114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>r2_scaled</th>\n",
       "      <td>0.862456</td>\n",
       "      <td>0.822291</td>\n",
       "      <td>0.891076</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    Tg         Td         Tm\n",
       "rmse         26.271093  44.638685  23.287833\n",
       "mae          18.618491  31.086444  16.986242\n",
       "r2            0.862456   0.822291   0.891076\n",
       "rmse_scaled   0.033129   0.050602   0.036114\n",
       "r2_scaled     0.862456   0.822291   0.891076"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(property_metric)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.7 ('.venv')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.7"
  },
  "output_auto_scroll": true,
  "vscode": {
   "interpreter": {
    "hash": "a686d76ec85f590216c9e7e08a09f9fc1c5b481b97ecdbc5e95356f9feef4f2c"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}