XGBOOST_feature_generation.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Libraries\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from rdkit import Chem\n",
    "from rdkit.ML.Descriptors.MoleculeDescriptors import MolecularDescriptorCalculator\n",
    "import mordred\n",
    "from mordred import Calculator, descriptors"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>al</th>\n",
       "      <th>CAS</th>\n",
       "      <th>smiles</th>\n",
       "      <th>δD</th>\n",
       "      <th>δP</th>\n",
       "      <th>δH</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1,1,1,2-Tetrachloroethane</td>\n",
       "      <td>b'630-20-6'</td>\n",
       "      <td>ClCC(Cl)(Cl)Cl</td>\n",
       "      <td>18.0</td>\n",
       "      <td>4.4</td>\n",
       "      <td>4.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1,1,1-Trichloroethane</td>\n",
       "      <td>b'71-55-6'</td>\n",
       "      <td>CC(Cl)(Cl)Cl</td>\n",
       "      <td>16.8</td>\n",
       "      <td>4.3</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1,1,1-Trifluoroethane</td>\n",
       "      <td>b'420-46-2'</td>\n",
       "      <td>CC(F)(F)F</td>\n",
       "      <td>14.6</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1,1,2,2-Tetrabromoethane</td>\n",
       "      <td>b'79-27-6'</td>\n",
       "      <td>BrC(Br)C(Br)Br</td>\n",
       "      <td>21.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1,1,2,2-Tetrachloroethane</td>\n",
       "      <td>b'79-34-5'</td>\n",
       "      <td>ClC(Cl)C(Cl)Cl</td>\n",
       "      <td>18.8</td>\n",
       "      <td>5.1</td>\n",
       "      <td>5.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1187</th>\n",
       "      <td>Quinine</td>\n",
       "      <td>b'130-95-0'</td>\n",
       "      <td>[H][C@@]1([C@@H](C2=CC=NC3=CC=C(C=C23)OC)O)C[C...</td>\n",
       "      <td>19.0</td>\n",
       "      <td>6.6</td>\n",
       "      <td>11.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1188</th>\n",
       "      <td>Sulfur Dioxide</td>\n",
       "      <td>b'9/5/7446'</td>\n",
       "      <td>O=S=O</td>\n",
       "      <td>15.8</td>\n",
       "      <td>8.4</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1189</th>\n",
       "      <td>Thionyl Chloride</td>\n",
       "      <td>b'9/7/7719'</td>\n",
       "      <td>O=S(Cl)Cl</td>\n",
       "      <td>16.9</td>\n",
       "      <td>6.4</td>\n",
       "      <td>6.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1190</th>\n",
       "      <td>Triethylene Glycol Monooleyl Ether</td>\n",
       "      <td>b'5274-66-8'</td>\n",
       "      <td>COCCOCCOCCO</td>\n",
       "      <td>16.0</td>\n",
       "      <td>3.1</td>\n",
       "      <td>8.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1191</th>\n",
       "      <td>Vinyl Silane</td>\n",
       "      <td>b'7291-09-0'</td>\n",
       "      <td>C=C[SiH3]</td>\n",
       "      <td>15.5</td>\n",
       "      <td>2.6</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1192 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      al           CAS   \n",
       "0              1,1,1,2-Tetrachloroethane   b'630-20-6'  \\\n",
       "1                  1,1,1-Trichloroethane    b'71-55-6'   \n",
       "2                  1,1,1-Trifluoroethane   b'420-46-2'   \n",
       "3               1,1,2,2-Tetrabromoethane    b'79-27-6'   \n",
       "4              1,1,2,2-Tetrachloroethane    b'79-34-5'   \n",
       "...                                  ...           ...   \n",
       "1187                             Quinine   b'130-95-0'   \n",
       "1188                      Sulfur Dioxide   b'9/5/7446'   \n",
       "1189                    Thionyl Chloride   b'9/7/7719'   \n",
       "1190  Triethylene Glycol Monooleyl Ether  b'5274-66-8'   \n",
       "1191                        Vinyl Silane  b'7291-09-0'   \n",
       "\n",
       "                                                 smiles    δD    δP    δH  \n",
       "0                                        ClCC(Cl)(Cl)Cl  18.0   4.4   4.2  \n",
       "1                                          CC(Cl)(Cl)Cl  16.8   4.3   2.0  \n",
       "2                                             CC(F)(F)F  14.6  10.0   0.0  \n",
       "3                                        BrC(Br)C(Br)Br  21.0   7.0   8.2  \n",
       "4                                        ClC(Cl)C(Cl)Cl  18.8   5.1   5.3  \n",
       "...                                                 ...   ...   ...   ...  \n",
       "1187  [H][C@@]1([C@@H](C2=CC=NC3=CC=C(C=C23)OC)O)C[C...  19.0   6.6  11.0  \n",
       "1188                                              O=S=O  15.8   8.4  10.0  \n",
       "1189                                          O=S(Cl)Cl  16.9   6.4   6.1  \n",
       "1190                                        COCCOCCOCCO  16.0   3.1   8.4  \n",
       "1191                                          C=C[SiH3]  15.5   2.6   4.0  \n",
       "\n",
       "[1192 rows x 6 columns]"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_csv('data/HSP_SMILES.csv',index_col=0)\n",
    "df.reset_index(drop=True,inplace=True) #since we removed 8 molecules such as: water , N , [Br-], etc. that can't be turned to graph molecules, we reset the indexes in the dataset \n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[14:53:43] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:53:43] WARNING: not removing hydrogen atom without neighbors\n"
     ]
    }
   ],
   "source": [
    "df['smiles'] = df['smiles'].astype(str)\n",
    "\n",
    "#get smiles in a list\n",
    "smiles=list(df['smiles'].values.astype(str))\n",
    "\n",
    "#generate molecule objects from smiles list\n",
    "mols = [Chem.MolFromSmiles(smile) for smile in smiles]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Descriptors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Initialize descritpor calculator:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1613"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calc = Calculator(descriptors,ignore_3D=True) \n",
    "len(calc.descriptors)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate descritpors for molecules in `mols`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54%|█████▍    | 648/1192 [00:09<00:05, 96.64it/s] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "c:\\Users\\darja\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\numpy\\core\\fromnumeric.py:86: RuntimeWarning: overflow encountered in reduce\n",
      "  return ufunc.reduce(obj, axis, dtype, out, **passkwargs)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62%|██████▏   | 742/1192 [00:11<00:06, 70.86it/s] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "c:\\Users\\darja\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\numpy\\core\\fromnumeric.py:86: RuntimeWarning: overflow encountered in reduce\n",
      "  return ufunc.reduce(obj, axis, dtype, out, **passkwargs)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|███████▊  | 932/1192 [00:13<00:03, 84.28it/s] [14:54:00] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:54:00] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:54:00] WARNING: not removing hydrogen atom without neighbors\n",
      " 83%|████████▎ | 985/1192 [00:14<00:02, 69.53it/s][14:54:01] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:54:01] WARNING: not removing hydrogen atom without neighbors\n",
      "[14:54:01] WARNING: not removing hydrogen atom without neighbors\n",
      "100%|██████████| 1192/1192 [00:16<00:00, 73.28it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'mordred._base.pandas_module.MordredDataFrame'>\n",
      "RangeIndex: 1192 entries, 0 to 1191\n",
      "Columns: 1613 entries, ABC to mZagreb2\n",
      "dtypes: bool(2), float64(500), int64(309), object(802)\n",
      "memory usage: 14.7+ MB\n"
     ]
    }
   ],
   "source": [
    "descs = calc.pandas(mols) \n",
    "descs.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Error tracking:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "ERRS = [mordred.error.MissingValueBase,\n",
    "        mordred.error.Missing,\n",
    "        mordred.error.Error,\n",
    "        mordred.error.MultipleFragments,\n",
    "        mordred.error.Missing3DCoordinate,\n",
    "        mordred.error.Timeout]\n",
    "\n",
    "# if there is an error, replace it with np.nan \n",
    "for err in ERRS:\n",
    "    descs = descs.applymap(lambda x: np.nan if isinstance(x, err) else x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The descriptor dataframe:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ABC</th>\n",
       "      <th>ABCGG</th>\n",
       "      <th>nAcid</th>\n",
       "      <th>nBase</th>\n",
       "      <th>SpAbs_A</th>\n",
       "      <th>SpMax_A</th>\n",
       "      <th>SpDiam_A</th>\n",
       "      <th>SpAD_A</th>\n",
       "      <th>SpMAD_A</th>\n",
       "      <th>LogEE_A</th>\n",
       "      <th>...</th>\n",
       "      <th>SRW10</th>\n",
       "      <th>TSRW10</th>\n",
       "      <th>MW</th>\n",
       "      <th>AMW</th>\n",
       "      <th>WPath</th>\n",
       "      <th>WPol</th>\n",
       "      <th>Zagreb1</th>\n",
       "      <th>Zagreb2</th>\n",
       "      <th>mZagreb1</th>\n",
       "      <th>mZagreb2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4.012290</td>\n",
       "      <td>4.284816</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5.818626</td>\n",
       "      <td>2.074313</td>\n",
       "      <td>4.148627</td>\n",
       "      <td>5.818626</td>\n",
       "      <td>0.969771</td>\n",
       "      <td>2.626363</td>\n",
       "      <td>...</td>\n",
       "      <td>7.989899</td>\n",
       "      <td>31.665095</td>\n",
       "      <td>165.891061</td>\n",
       "      <td>20.736383</td>\n",
       "      <td>28</td>\n",
       "      <td>3</td>\n",
       "      <td>24.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>4.312500</td>\n",
       "      <td>1.375000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.464102</td>\n",
       "      <td>3.464102</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>2.444466</td>\n",
       "      <td>...</td>\n",
       "      <td>7.625107</td>\n",
       "      <td>29.418928</td>\n",
       "      <td>131.930033</td>\n",
       "      <td>16.491254</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>4.062500</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.464102</td>\n",
       "      <td>3.464102</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>2.444466</td>\n",
       "      <td>...</td>\n",
       "      <td>7.625107</td>\n",
       "      <td>29.418928</td>\n",
       "      <td>84.018685</td>\n",
       "      <td>10.502336</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>4.062500</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.932653</td>\n",
       "      <td>4.244375</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.610845</td>\n",
       "      <td>...</td>\n",
       "      <td>7.626083</td>\n",
       "      <td>30.698690</td>\n",
       "      <td>341.688998</td>\n",
       "      <td>42.711125</td>\n",
       "      <td>29</td>\n",
       "      <td>4</td>\n",
       "      <td>22.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>4.222222</td>\n",
       "      <td>1.444444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.932653</td>\n",
       "      <td>4.244375</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.610845</td>\n",
       "      <td>...</td>\n",
       "      <td>7.626083</td>\n",
       "      <td>30.698690</td>\n",
       "      <td>165.891061</td>\n",
       "      <td>20.736383</td>\n",
       "      <td>29</td>\n",
       "      <td>4</td>\n",
       "      <td>22.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>4.222222</td>\n",
       "      <td>1.444444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1187</th>\n",
       "      <td>18.958632</td>\n",
       "      <td>14.851657</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>32.595631</td>\n",
       "      <td>2.494948</td>\n",
       "      <td>4.989896</td>\n",
       "      <td>32.595631</td>\n",
       "      <td>1.358151</td>\n",
       "      <td>4.132305</td>\n",
       "      <td>...</td>\n",
       "      <td>10.353448</td>\n",
       "      <td>59.266606</td>\n",
       "      <td>324.183778</td>\n",
       "      <td>6.753829</td>\n",
       "      <td>1286</td>\n",
       "      <td>42</td>\n",
       "      <td>132.0</td>\n",
       "      <td>161.0</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>5.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1188</th>\n",
       "      <td>1.414214</td>\n",
       "      <td>1.414214</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.828427</td>\n",
       "      <td>1.414214</td>\n",
       "      <td>2.828427</td>\n",
       "      <td>2.828427</td>\n",
       "      <td>0.942809</td>\n",
       "      <td>1.849457</td>\n",
       "      <td>...</td>\n",
       "      <td>4.174387</td>\n",
       "      <td>17.310771</td>\n",
       "      <td>63.961900</td>\n",
       "      <td>21.320633</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.250000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1189</th>\n",
       "      <td>2.449490</td>\n",
       "      <td>2.449490</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3.464102</td>\n",
       "      <td>1.732051</td>\n",
       "      <td>3.464102</td>\n",
       "      <td>3.464102</td>\n",
       "      <td>0.866025</td>\n",
       "      <td>2.178059</td>\n",
       "      <td>...</td>\n",
       "      <td>6.188264</td>\n",
       "      <td>24.179697</td>\n",
       "      <td>117.904691</td>\n",
       "      <td>29.476173</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>3.111111</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1190</th>\n",
       "      <td>7.071068</td>\n",
       "      <td>6.765664</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13.191508</td>\n",
       "      <td>1.931852</td>\n",
       "      <td>3.863703</td>\n",
       "      <td>13.191508</td>\n",
       "      <td>1.199228</td>\n",
       "      <td>3.202455</td>\n",
       "      <td>...</td>\n",
       "      <td>7.601402</td>\n",
       "      <td>37.236738</td>\n",
       "      <td>164.104859</td>\n",
       "      <td>6.077958</td>\n",
       "      <td>220</td>\n",
       "      <td>8</td>\n",
       "      <td>38.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>4.250000</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1191</th>\n",
       "      <td>1.414214</td>\n",
       "      <td>1.414214</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.828427</td>\n",
       "      <td>1.414214</td>\n",
       "      <td>2.828427</td>\n",
       "      <td>2.828427</td>\n",
       "      <td>0.942809</td>\n",
       "      <td>1.849457</td>\n",
       "      <td>...</td>\n",
       "      <td>4.174387</td>\n",
       "      <td>17.310771</td>\n",
       "      <td>58.023877</td>\n",
       "      <td>6.447097</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.250000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1192 rows × 1613 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            ABC      ABCGG  nAcid  nBase    SpAbs_A   SpMax_A  SpDiam_A   \n",
       "0      4.012290   4.284816      0      0   5.818626  2.074313  4.148627  \\\n",
       "1      3.464102   3.464102      0      0   4.000000  2.000000  4.000000   \n",
       "2      3.464102   3.464102      0      0   4.000000  2.000000  4.000000   \n",
       "3      3.932653   4.244375      0      0   6.000000  2.000000  4.000000   \n",
       "4      3.932653   4.244375      0      0   6.000000  2.000000  4.000000   \n",
       "...         ...        ...    ...    ...        ...       ...       ...   \n",
       "1187  18.958632  14.851657      0      1  32.595631  2.494948  4.989896   \n",
       "1188   1.414214   1.414214      0      0   2.828427  1.414214  2.828427   \n",
       "1189   2.449490   2.449490      0      0   3.464102  1.732051  3.464102   \n",
       "1190   7.071068   6.765664      0      0  13.191508  1.931852  3.863703   \n",
       "1191   1.414214   1.414214      0      0   2.828427  1.414214  2.828427   \n",
       "\n",
       "         SpAD_A   SpMAD_A   LogEE_A  ...      SRW10     TSRW10          MW   \n",
       "0      5.818626  0.969771  2.626363  ...   7.989899  31.665095  165.891061  \\\n",
       "1      4.000000  0.800000  2.444466  ...   7.625107  29.418928  131.930033   \n",
       "2      4.000000  0.800000  2.444466  ...   7.625107  29.418928   84.018685   \n",
       "3      6.000000  1.000000  2.610845  ...   7.626083  30.698690  341.688998   \n",
       "4      6.000000  1.000000  2.610845  ...   7.626083  30.698690  165.891061   \n",
       "...         ...       ...       ...  ...        ...        ...         ...   \n",
       "1187  32.595631  1.358151  4.132305  ...  10.353448  59.266606  324.183778   \n",
       "1188   2.828427  0.942809  1.849457  ...   4.174387  17.310771   63.961900   \n",
       "1189   3.464102  0.866025  2.178059  ...   6.188264  24.179697  117.904691   \n",
       "1190  13.191508  1.199228  3.202455  ...   7.601402  37.236738  164.104859   \n",
       "1191   2.828427  0.942809  1.849457  ...   4.174387  17.310771   58.023877   \n",
       "\n",
       "            AMW  WPath  WPol  Zagreb1  Zagreb2  mZagreb1  mZagreb2  \n",
       "0     20.736383     28     3     24.0     22.0  4.312500  1.375000  \n",
       "1     16.491254     16     0     20.0     16.0  4.062500  1.000000  \n",
       "2     10.502336     16     0     20.0     16.0  4.062500  1.000000  \n",
       "3     42.711125     29     4     22.0     21.0  4.222222  1.444444  \n",
       "4     20.736383     29     4     22.0     21.0  4.222222  1.444444  \n",
       "...         ...    ...   ...      ...      ...       ...       ...  \n",
       "1187   6.753829   1286    42    132.0    161.0  7.000000  5.333333  \n",
       "1188  21.320633      4     0      6.0      4.0  2.250000  1.000000  \n",
       "1189  29.476173      9     0     12.0      9.0  3.111111  1.000000  \n",
       "1190   6.077958    220     8     38.0     36.0  4.250000  3.000000  \n",
       "1191   6.447097      4     0      6.0      4.0  2.250000  1.000000  \n",
       "\n",
       "[1192 rows x 1613 columns]"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "descs"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Count the NaN values (errors) in columns:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "nans = {}\n",
    "for column_name in descs.columns:\n",
    "    nans[column_name]=0\n",
    "    \n",
    "for column_name in descs.columns:\n",
    "    column = descs[column_name]\n",
    "    # Get the count of nans in column \n",
    "    count = (column.isna()).sum()\n",
    "    nans[column_name]=count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nans</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ABC</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABCGG</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nAcid</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nBase</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SpAbs_A</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WPol</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Zagreb1</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Zagreb2</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mZagreb1</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mZagreb2</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1613 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          nans\n",
       "ABC          0\n",
       "ABCGG        0\n",
       "nAcid        0\n",
       "nBase        0\n",
       "SpAbs_A      4\n",
       "...        ...\n",
       "WPol         0\n",
       "Zagreb1      0\n",
       "Zagreb2      0\n",
       "mZagreb1     3\n",
       "mZagreb2     0\n",
       "\n",
       "[1613 rows x 1 columns]"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#make dataframe from dict to see the amount of NaN values for each descriptor\n",
    "nans = pd.DataFrame.from_dict(nans,orient='index',columns=['nans'])\n",
    "nans"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Select descriptors that have more than 5% of NaN values for our dataset:\n",
    "\n",
    "(5% is arbitrary, but since the dataset is smaller, it's better to clear out as much errors as possible)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nans</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AATS4dv</th>\n",
       "      <td>127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AATS5dv</th>\n",
       "      <td>288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AATS6dv</th>\n",
       "      <td>475</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AATS7dv</th>\n",
       "      <td>697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AATS8dv</th>\n",
       "      <td>835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MDEN-12</th>\n",
       "      <td>1180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MDEN-13</th>\n",
       "      <td>1189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MDEN-22</th>\n",
       "      <td>1176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MDEN-23</th>\n",
       "      <td>1187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MDEN-33</th>\n",
       "      <td>1175</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>437 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         nans\n",
       "AATS4dv   127\n",
       "AATS5dv   288\n",
       "AATS6dv   475\n",
       "AATS7dv   697\n",
       "AATS8dv   835\n",
       "...       ...\n",
       "MDEN-12  1180\n",
       "MDEN-13  1189\n",
       "MDEN-22  1176\n",
       "MDEN-23  1187\n",
       "MDEN-33  1175\n",
       "\n",
       "[437 rows x 1 columns]"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "threshold = 0.05 #5%\n",
    "n=descs.shape[0]\n",
    "t=threshold*n #%5 in numbers \n",
    "\n",
    "nans[nans['nans']>=t]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove the descriptors that satisfy this condition:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "437\n"
     ]
    }
   ],
   "source": [
    "nans_to_drop = nans[nans['nans']>=t].index\n",
    "print(len(nans_to_drop))\n",
    "#new descriptor dataframe\n",
    "descs_nonan=descs.drop(nans_to_drop,axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, count the number of zeros for each descriptor. We remove the descriptor if it has over 90% of zero values:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "305\n"
     ]
    }
   ],
   "source": [
    "zeros = {}\n",
    "\n",
    "for column_name in descs_nonan.columns:\n",
    "    zeros[column_name]=0\n",
    "    \n",
    "for column_name in descs_nonan.columns:\n",
    "    column = descs_nonan[column_name]\n",
    "    # Get the count of Zeros in column \n",
    "    count = (column == 0).sum()\n",
    "    zeros[column_name]=count\n",
    "\n",
    "zeros = pd.DataFrame.from_dict(zeros,orient='index',columns=['zeros'])\n",
    "threshold = 0.9\n",
    "n=descs_nonan.shape[0]\n",
    "t=threshold*n\n",
    "\n",
    "zeros_to_drop=zeros[zeros['zeros']>=t].index\n",
    "print(len(zeros_to_drop))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Drop these columns(descriptors):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "descs_filtered = descs_nonan.drop(zeros_to_drop,axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ABC</th>\n",
       "      <th>ABCGG</th>\n",
       "      <th>SpAbs_A</th>\n",
       "      <th>SpMax_A</th>\n",
       "      <th>SpDiam_A</th>\n",
       "      <th>SpAD_A</th>\n",
       "      <th>SpMAD_A</th>\n",
       "      <th>LogEE_A</th>\n",
       "      <th>VE1_A</th>\n",
       "      <th>VE2_A</th>\n",
       "      <th>...</th>\n",
       "      <th>SRW10</th>\n",
       "      <th>TSRW10</th>\n",
       "      <th>MW</th>\n",
       "      <th>AMW</th>\n",
       "      <th>WPath</th>\n",
       "      <th>WPol</th>\n",
       "      <th>Zagreb1</th>\n",
       "      <th>Zagreb2</th>\n",
       "      <th>mZagreb1</th>\n",
       "      <th>mZagreb2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4.012290</td>\n",
       "      <td>4.284816</td>\n",
       "      <td>5.818626</td>\n",
       "      <td>2.074313</td>\n",
       "      <td>4.148627</td>\n",
       "      <td>5.818626</td>\n",
       "      <td>0.969771</td>\n",
       "      <td>2.626363</td>\n",
       "      <td>2.285500</td>\n",
       "      <td>0.380917</td>\n",
       "      <td>...</td>\n",
       "      <td>7.989899</td>\n",
       "      <td>31.665095</td>\n",
       "      <td>165.891061</td>\n",
       "      <td>20.736383</td>\n",
       "      <td>28</td>\n",
       "      <td>3</td>\n",
       "      <td>24.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>4.312500</td>\n",
       "      <td>1.375000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.464102</td>\n",
       "      <td>3.464102</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>2.444466</td>\n",
       "      <td>2.121320</td>\n",
       "      <td>0.424264</td>\n",
       "      <td>...</td>\n",
       "      <td>7.625107</td>\n",
       "      <td>29.418928</td>\n",
       "      <td>131.930033</td>\n",
       "      <td>16.491254</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>4.062500</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.464102</td>\n",
       "      <td>3.464102</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>2.444466</td>\n",
       "      <td>2.121320</td>\n",
       "      <td>0.424264</td>\n",
       "      <td>...</td>\n",
       "      <td>7.625107</td>\n",
       "      <td>29.418928</td>\n",
       "      <td>84.018685</td>\n",
       "      <td>10.502336</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>4.062500</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.932653</td>\n",
       "      <td>4.244375</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.610845</td>\n",
       "      <td>2.309401</td>\n",
       "      <td>0.384900</td>\n",
       "      <td>...</td>\n",
       "      <td>7.626083</td>\n",
       "      <td>30.698690</td>\n",
       "      <td>341.688998</td>\n",
       "      <td>42.711125</td>\n",
       "      <td>29</td>\n",
       "      <td>4</td>\n",
       "      <td>22.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>4.222222</td>\n",
       "      <td>1.444444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.932653</td>\n",
       "      <td>4.244375</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.610845</td>\n",
       "      <td>2.309401</td>\n",
       "      <td>0.384900</td>\n",
       "      <td>...</td>\n",
       "      <td>7.626083</td>\n",
       "      <td>30.698690</td>\n",
       "      <td>165.891061</td>\n",
       "      <td>20.736383</td>\n",
       "      <td>29</td>\n",
       "      <td>4</td>\n",
       "      <td>22.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>4.222222</td>\n",
       "      <td>1.444444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1187</th>\n",
       "      <td>18.958632</td>\n",
       "      <td>14.851657</td>\n",
       "      <td>32.595631</td>\n",
       "      <td>2.494948</td>\n",
       "      <td>4.989896</td>\n",
       "      <td>32.595631</td>\n",
       "      <td>1.358151</td>\n",
       "      <td>4.132305</td>\n",
       "      <td>4.427587</td>\n",
       "      <td>0.184483</td>\n",
       "      <td>...</td>\n",
       "      <td>10.353448</td>\n",
       "      <td>59.266606</td>\n",
       "      <td>324.183778</td>\n",
       "      <td>6.753829</td>\n",
       "      <td>1286</td>\n",
       "      <td>42</td>\n",
       "      <td>132.0</td>\n",
       "      <td>161.0</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>5.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1188</th>\n",
       "      <td>1.414214</td>\n",
       "      <td>1.414214</td>\n",
       "      <td>2.828427</td>\n",
       "      <td>1.414214</td>\n",
       "      <td>2.828427</td>\n",
       "      <td>2.828427</td>\n",
       "      <td>0.942809</td>\n",
       "      <td>1.849457</td>\n",
       "      <td>1.707107</td>\n",
       "      <td>0.569036</td>\n",
       "      <td>...</td>\n",
       "      <td>4.174387</td>\n",
       "      <td>17.310771</td>\n",
       "      <td>63.961900</td>\n",
       "      <td>21.320633</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.250000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1189</th>\n",
       "      <td>2.449490</td>\n",
       "      <td>2.449490</td>\n",
       "      <td>3.464102</td>\n",
       "      <td>1.732051</td>\n",
       "      <td>3.464102</td>\n",
       "      <td>3.464102</td>\n",
       "      <td>0.866025</td>\n",
       "      <td>2.178059</td>\n",
       "      <td>1.931852</td>\n",
       "      <td>0.482963</td>\n",
       "      <td>...</td>\n",
       "      <td>6.188264</td>\n",
       "      <td>24.179697</td>\n",
       "      <td>117.904691</td>\n",
       "      <td>29.476173</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>3.111111</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1190</th>\n",
       "      <td>7.071068</td>\n",
       "      <td>6.765664</td>\n",
       "      <td>13.191508</td>\n",
       "      <td>1.931852</td>\n",
       "      <td>3.863703</td>\n",
       "      <td>13.191508</td>\n",
       "      <td>1.199228</td>\n",
       "      <td>3.202455</td>\n",
       "      <td>3.100954</td>\n",
       "      <td>0.281905</td>\n",
       "      <td>...</td>\n",
       "      <td>7.601402</td>\n",
       "      <td>37.236738</td>\n",
       "      <td>164.104859</td>\n",
       "      <td>6.077958</td>\n",
       "      <td>220</td>\n",
       "      <td>8</td>\n",
       "      <td>38.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>4.250000</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1191</th>\n",
       "      <td>1.414214</td>\n",
       "      <td>1.414214</td>\n",
       "      <td>2.828427</td>\n",
       "      <td>1.414214</td>\n",
       "      <td>2.828427</td>\n",
       "      <td>2.828427</td>\n",
       "      <td>0.942809</td>\n",
       "      <td>1.849457</td>\n",
       "      <td>1.707107</td>\n",
       "      <td>0.569036</td>\n",
       "      <td>...</td>\n",
       "      <td>4.174387</td>\n",
       "      <td>17.310771</td>\n",
       "      <td>58.023877</td>\n",
       "      <td>6.447097</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.250000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1192 rows × 871 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            ABC      ABCGG    SpAbs_A   SpMax_A  SpDiam_A     SpAD_A   \n",
       "0      4.012290   4.284816   5.818626  2.074313  4.148627   5.818626  \\\n",
       "1      3.464102   3.464102   4.000000  2.000000  4.000000   4.000000   \n",
       "2      3.464102   3.464102   4.000000  2.000000  4.000000   4.000000   \n",
       "3      3.932653   4.244375   6.000000  2.000000  4.000000   6.000000   \n",
       "4      3.932653   4.244375   6.000000  2.000000  4.000000   6.000000   \n",
       "...         ...        ...        ...       ...       ...        ...   \n",
       "1187  18.958632  14.851657  32.595631  2.494948  4.989896  32.595631   \n",
       "1188   1.414214   1.414214   2.828427  1.414214  2.828427   2.828427   \n",
       "1189   2.449490   2.449490   3.464102  1.732051  3.464102   3.464102   \n",
       "1190   7.071068   6.765664  13.191508  1.931852  3.863703  13.191508   \n",
       "1191   1.414214   1.414214   2.828427  1.414214  2.828427   2.828427   \n",
       "\n",
       "       SpMAD_A   LogEE_A     VE1_A     VE2_A  ...      SRW10     TSRW10   \n",
       "0     0.969771  2.626363  2.285500  0.380917  ...   7.989899  31.665095  \\\n",
       "1     0.800000  2.444466  2.121320  0.424264  ...   7.625107  29.418928   \n",
       "2     0.800000  2.444466  2.121320  0.424264  ...   7.625107  29.418928   \n",
       "3     1.000000  2.610845  2.309401  0.384900  ...   7.626083  30.698690   \n",
       "4     1.000000  2.610845  2.309401  0.384900  ...   7.626083  30.698690   \n",
       "...        ...       ...       ...       ...  ...        ...        ...   \n",
       "1187  1.358151  4.132305  4.427587  0.184483  ...  10.353448  59.266606   \n",
       "1188  0.942809  1.849457  1.707107  0.569036  ...   4.174387  17.310771   \n",
       "1189  0.866025  2.178059  1.931852  0.482963  ...   6.188264  24.179697   \n",
       "1190  1.199228  3.202455  3.100954  0.281905  ...   7.601402  37.236738   \n",
       "1191  0.942809  1.849457  1.707107  0.569036  ...   4.174387  17.310771   \n",
       "\n",
       "              MW        AMW  WPath  WPol  Zagreb1  Zagreb2  mZagreb1  mZagreb2  \n",
       "0     165.891061  20.736383     28     3     24.0     22.0  4.312500  1.375000  \n",
       "1     131.930033  16.491254     16     0     20.0     16.0  4.062500  1.000000  \n",
       "2      84.018685  10.502336     16     0     20.0     16.0  4.062500  1.000000  \n",
       "3     341.688998  42.711125     29     4     22.0     21.0  4.222222  1.444444  \n",
       "4     165.891061  20.736383     29     4     22.0     21.0  4.222222  1.444444  \n",
       "...          ...        ...    ...   ...      ...      ...       ...       ...  \n",
       "1187  324.183778   6.753829   1286    42    132.0    161.0  7.000000  5.333333  \n",
       "1188   63.961900  21.320633      4     0      6.0      4.0  2.250000  1.000000  \n",
       "1189  117.904691  29.476173      9     0     12.0      9.0  3.111111  1.000000  \n",
       "1190  164.104859   6.077958    220     8     38.0     36.0  4.250000  3.000000  \n",
       "1191   58.023877   6.447097      4     0      6.0      4.0  2.250000  1.000000  \n",
       "\n",
       "[1192 rows x 871 columns]"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "descs_filtered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>al</th>\n",
       "      <th>CAS</th>\n",
       "      <th>smiles</th>\n",
       "      <th>ABC</th>\n",
       "      <th>ABCGG</th>\n",
       "      <th>SpAbs_A</th>\n",
       "      <th>SpMax_A</th>\n",
       "      <th>SpDiam_A</th>\n",
       "      <th>SpAD_A</th>\n",
       "      <th>SpMAD_A</th>\n",
       "      <th>...</th>\n",
       "      <th>AMW</th>\n",
       "      <th>WPath</th>\n",
       "      <th>WPol</th>\n",
       "      <th>Zagreb1</th>\n",
       "      <th>Zagreb2</th>\n",
       "      <th>mZagreb1</th>\n",
       "      <th>mZagreb2</th>\n",
       "      <th>δD</th>\n",
       "      <th>δP</th>\n",
       "      <th>δH</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1,1,1,2-Tetrachloroethane</td>\n",
       "      <td>b'630-20-6'</td>\n",
       "      <td>ClCC(Cl)(Cl)Cl</td>\n",
       "      <td>4.012290</td>\n",
       "      <td>4.284816</td>\n",
       "      <td>5.818626</td>\n",
       "      <td>2.074313</td>\n",
       "      <td>4.148627</td>\n",
       "      <td>5.818626</td>\n",
       "      <td>0.969771</td>\n",
       "      <td>...</td>\n",
       "      <td>20.736383</td>\n",
       "      <td>28</td>\n",
       "      <td>3</td>\n",
       "      <td>24.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>4.312500</td>\n",
       "      <td>1.375000</td>\n",
       "      <td>18.0</td>\n",
       "      <td>4.4</td>\n",
       "      <td>4.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1,1,1-Trichloroethane</td>\n",
       "      <td>b'71-55-6'</td>\n",
       "      <td>CC(Cl)(Cl)Cl</td>\n",
       "      <td>3.464102</td>\n",
       "      <td>3.464102</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>...</td>\n",
       "      <td>16.491254</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>4.062500</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>16.8</td>\n",
       "      <td>4.3</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1,1,1-Trifluoroethane</td>\n",
       "      <td>b'420-46-2'</td>\n",
       "      <td>CC(F)(F)F</td>\n",
       "      <td>3.464102</td>\n",
       "      <td>3.464102</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>...</td>\n",
       "      <td>10.502336</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>4.062500</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>14.6</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1,1,2,2-Tetrabromoethane</td>\n",
       "      <td>b'79-27-6'</td>\n",
       "      <td>BrC(Br)C(Br)Br</td>\n",
       "      <td>3.932653</td>\n",
       "      <td>4.244375</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>42.711125</td>\n",
       "      <td>29</td>\n",
       "      <td>4</td>\n",
       "      <td>22.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>4.222222</td>\n",
       "      <td>1.444444</td>\n",
       "      <td>21.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1,1,2,2-Tetrachloroethane</td>\n",
       "      <td>b'79-34-5'</td>\n",
       "      <td>ClC(Cl)C(Cl)Cl</td>\n",
       "      <td>3.932653</td>\n",
       "      <td>4.244375</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>20.736383</td>\n",
       "      <td>29</td>\n",
       "      <td>4</td>\n",
       "      <td>22.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>4.222222</td>\n",
       "      <td>1.444444</td>\n",
       "      <td>18.8</td>\n",
       "      <td>5.1</td>\n",
       "      <td>5.3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 877 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                          al          CAS          smiles       ABC     ABCGG   \n",
       "0  1,1,1,2-Tetrachloroethane  b'630-20-6'  ClCC(Cl)(Cl)Cl  4.012290  4.284816  \\\n",
       "1      1,1,1-Trichloroethane   b'71-55-6'    CC(Cl)(Cl)Cl  3.464102  3.464102   \n",
       "2      1,1,1-Trifluoroethane  b'420-46-2'       CC(F)(F)F  3.464102  3.464102   \n",
       "3   1,1,2,2-Tetrabromoethane   b'79-27-6'  BrC(Br)C(Br)Br  3.932653  4.244375   \n",
       "4  1,1,2,2-Tetrachloroethane   b'79-34-5'  ClC(Cl)C(Cl)Cl  3.932653  4.244375   \n",
       "\n",
       "    SpAbs_A   SpMax_A  SpDiam_A    SpAD_A   SpMAD_A  ...        AMW  WPath   \n",
       "0  5.818626  2.074313  4.148627  5.818626  0.969771  ...  20.736383     28  \\\n",
       "1  4.000000  2.000000  4.000000  4.000000  0.800000  ...  16.491254     16   \n",
       "2  4.000000  2.000000  4.000000  4.000000  0.800000  ...  10.502336     16   \n",
       "3  6.000000  2.000000  4.000000  6.000000  1.000000  ...  42.711125     29   \n",
       "4  6.000000  2.000000  4.000000  6.000000  1.000000  ...  20.736383     29   \n",
       "\n",
       "   WPol  Zagreb1  Zagreb2  mZagreb1  mZagreb2    δD    δP   δH  \n",
       "0     3     24.0     22.0  4.312500  1.375000  18.0   4.4  4.2  \n",
       "1     0     20.0     16.0  4.062500  1.000000  16.8   4.3  2.0  \n",
       "2     0     20.0     16.0  4.062500  1.000000  14.6  10.0  0.0  \n",
       "3     4     22.0     21.0  4.222222  1.444444  21.0   7.0  8.2  \n",
       "4     4     22.0     21.0  4.222222  1.444444  18.8   5.1  5.3  \n",
       "\n",
       "[5 rows x 877 columns]"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DF=pd.concat([df,descs_filtered],axis=1)\n",
    "\n",
    "new_cols = list(DF.columns)\n",
    "trm=['δD', 'δP', 'δH']\n",
    "\n",
    "for el in trm:\n",
    "    new_cols.remove(el)\n",
    "    \n",
    "new_cols+=trm\n",
    "DF = DF[new_cols]\n",
    "\n",
    "DF.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "DF.drop(['Lipinski','GhoseFilter'],axis=1,inplace=True) #this descriptors raise some errors in the code, so we remove them"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "DF.to_csv('data/HSP_descriptors_NO_ERRS_ZEROS.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "5ac50a45280b8a64f634f3215b823254991aa4a8c8ad1726e4d2ca2e43196b44"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}