004. Similarity search performance evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f2a2f747",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "from random import sample\n",
    "from statistics import mean\n",
    "from sklearn.metrics import roc_curve, roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "56235f5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def CI(stats):\n",
    "    # plot scores\n",
    "    plt.hist(stats)\n",
    "    plt.show()\n",
    "    \n",
    "    # Confidence intervals\n",
    "    alpha = 0.95\n",
    "    # Lower and Upper\n",
    "    p1 = ((1.0-alpha)/2.0) * 100  \n",
    "    lower = max(0.0, np.percentile(stats, p1))\n",
    "    p2 = (alpha+((1.0-alpha)/2.0)) * 100\n",
    "    upper = min(1.0, np.percentile(stats, p2))\n",
    "    \n",
    "    print('%.1f confidence interval of MCC %.3f and %.3f' % (alpha*100, lower, upper))\n",
    "    print('Average MCC %.3f' % np.mean(stats))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4d8cf12c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tanimoto coefficient\n",
    "def tanimoto(fingerprint1, fingerprint2):\n",
    "    fp1 = int(fingerprint1, 16)\n",
    "    fp2 = int(fingerprint2, 16)\n",
    "    fp1_count = bin(fp1).count('1')\n",
    "    fp2_count = bin(fp2).count('1')\n",
    "    both_count = bin(fp1 & fp2).count('1')  \n",
    "    \n",
    "    if (fp1_count + fp2_count - both_count) != 0:\n",
    "        return float(both_count)/(fp1_count + fp2_count - both_count)\n",
    "    else:\n",
    "        return float(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "db3e2606",
   "metadata": {},
   "outputs": [],
   "source": [
    "def rogot_goldberg(fingerprint1, fingerprint2):\n",
    "    sum1, sum2 = 0,0\n",
    "    a,b,c,d = 0,0,0,0 \n",
    "    for i in range(len(fingerprint1)):\n",
    "        if fingerprint1[i] == \"1\" and fingerprint2[i] == \"1\":\n",
    "            a += 1\n",
    "            \n",
    "        elif fingerprint1[i] == \"1\" and fingerprint2[i] == \"0\":\n",
    "            b += 1\n",
    "            \n",
    "        elif fingerprint1[i] == \"0\" and fingerprint2[i] == \"1\":\n",
    "            c += 1\n",
    "           \n",
    "        elif fingerprint1[i] == \"0\" and fingerprint2[i] == \"0\":\n",
    "            d += 1\n",
    "    try:\n",
    "        sum1 = a/(2*a+b+c)\n",
    "    except ZeroDivisionError:\n",
    "        sum1 = 0 \n",
    "        \n",
    "    try:\n",
    "        sum2 = d/(2*d+b+c)\n",
    "    except ZeroDivisionError:\n",
    "        sum2 = 0 \n",
    "\n",
    "    return sum1+sum2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8c53348d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_opt_thres(fpr, tpr, thresholds):\n",
    "    # Calculate the G-mean\n",
    "    gmean = np.sqrt(tpr*(1-fpr))\n",
    "    # Maximal G-mean corresponding index\n",
    "    index = np.argmax(gmean)\n",
    "    # get optimal threshold corrsponding to maximal G-mean\n",
    "    thresholdOpt = round(thresholds[index], ndigits=4)\n",
    "    return thresholdOpt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bdd6579e",
   "metadata": {},
   "source": [
    "## Similarity search performance evaluation\n",
    "Metrics: AUC and EF10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0e85fd0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '/Users/hek/Research/Cheminformatics/Project_1_NPS/Stimulant vs. Hallucinogens/ChEMBL Dataset ML results/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "89666332",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Depressants_Opioids' 'Stimulants' 'Cannabinoids'\n",
      " 'Serotonergic psychedelics' 'Depressants_Benzodiazepines']\n"
     ]
    }
   ],
   "source": [
    "NPS_active = pd.read_csv(path+\"Drugs Raman or SERS in literature _ Paper 1.csv\",dtype=\"object\")\n",
    "NPS_active = NPS_active.astype({\"Pharm class label\": \"float64\"})\n",
    "NPS_inactive = pd.read_csv(path+\"Decoys Drugs Raman or SERS in literature _ Paper 1.csv\",dtype=\"object\")\n",
    "print(NPS_active['Pharm class'].unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7f9f89d0",
   "metadata": {},
   "source": [
    "### Part I. Similarity search for each Pharm Category\n",
    "* 1 == 'Depressants_Opioids'\n",
    "* 2 == 'Stimulants'\n",
    "* 3 == 'Cannabinoids'\n",
    "* 4 == 'Serotonergic psychedelics'\n",
    "* 5 == 'Depressants_Benzodiazepines'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e50c2596",
   "metadata": {},
   "source": [
    "#### 1. Assemble query list (training) and test list molecules for each pharm category\n",
    "##### Repeat similarity search for 50 times\n",
    "##### Each iteration: \n",
    "* random sample 10 actives, and 20% of the decoys as query list\n",
    "* remaining active and decoys as test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f2fb138f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def actives_query_test_split(pharm_class):\n",
    "    # Subset actives by pharm class label according to category, reindex dataframe\n",
    "    actives = NPS_active[NPS_active['Pharm class label']==pharm_class]\n",
    "    #print(\"Total of\", actives.shape[0], \"actives in pharm class = \",pharm_class)\n",
    "    actives.reset_index(drop=True,inplace=True)\n",
    "    actives_tmp = actives.copy()\n",
    "    # Random sample 10 active by index\n",
    "    random10_active = sample(range(actives.shape[0]), 10)\n",
    "    # subset by random active index\n",
    "    actives_query = actives.iloc[random10_active,:]\n",
    "    #print(actives_query.Name)\n",
    "\n",
    "    # drop sampled actives by index\n",
    "    actives_tmp.drop(random10_active,inplace=True)\n",
    "    #print(\"Number actives added to test list\", P_total = actives_tmp.shape[0])\n",
    "    P_total = actives_tmp.shape[0]\n",
    "    n_size = P_total*10\n",
    "    #print(\"Number of decoys to be added to test list\",n_size)\n",
    "    return actives_query, actives_tmp,P_total, n_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a6977ead",
   "metadata": {},
   "outputs": [],
   "source": [
    "def decoys_query_test_split(n_size):\n",
    "    decoys_tmp = NPS_inactive.copy()\n",
    "    decoys_tmp['Pharm class label']=np.zeros(len(decoys_tmp))\n",
    "    #print(decoys_tmp.shape[0])\n",
    "    # Random sample 100 decoys by index\n",
    "    random_decoy = sample(range(decoys_tmp.shape[0]),100)\n",
    "    # Subset by random decoy index\n",
    "    decoys_query = decoys_tmp.iloc[random_decoy,:]\n",
    "    #print(\"Number of decoys added to query\", decoys_query.shape[0])\n",
    "    # Drop sampled qeury decoys by index, and reset\n",
    "    decoys_tmp.drop(random_decoy,inplace=True)\n",
    "    decoys_tmp.reset_index(drop=True, inplace=True)\n",
    "    \n",
    "    # Random sample n_size = (query actives)*10\n",
    "    random_decoy_test = sample(range(decoys_tmp.shape[0]),n_size)\n",
    "    decoys_test = decoys_tmp.iloc[random_decoy_test,:]\n",
    "    #print(\"Number decoys added to test list\", decoys_test.shape[0])\n",
    "    \n",
    "    return decoys_query, decoys_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d2f17aa3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def merge_query_test(pharm_class):\n",
    "    actives_query, actives_test, P_total,n_size = actives_query_test_split(pharm_class)\n",
    "    decoys_query, decoys_test = decoys_query_test_split(n_size)\n",
    "    \n",
    "    df_query = pd.concat([actives_query,decoys_query],axis=0)\n",
    "    df_query.reset_index(drop=True, inplace=True)\n",
    "    #df_query = df_query[['Name','Pharm class label','maccsfp','morganfp']]\n",
    "    #print(df_query.shape[0])\n",
    "    df_test = pd.concat([actives_test,decoys_test],axis=0)\n",
    "    df_test.reset_index(drop=True, inplace=True)\n",
    "    #df_test = df_test[['Name','Pharm class label','maccsfp','morganfp']]\n",
    "    #print(df_test.shape[0])\n",
    "    N_total = df_test.shape[0]\n",
    "    #print(\"Test set P_total/N_total ratio\", P_total/N_total)\n",
    "    return df_query, df_test, P_total, N_total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "591b1ad7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def df_test_search_result(df_query, df_test,fingerprint):\n",
    "    df_test_search = df_test.copy()\n",
    "    # For each molecule in the test set, calculate its similarity to query molecules, and only the highest similarity value was considered\n",
    "    top_tani = []\n",
    "    for test in range(df_test.shape[0]):\n",
    "        #print(df_test.loc[test,\"Name\"])\n",
    "        #print(len(df_test.loc[0,fingerprint]))\n",
    "        tani = []\n",
    "        for query in (range(df_query.shape[0])):\n",
    "            #print(df_query.loc[query,\"Name\"])\n",
    "            tani.append(tanimoto(df_test.loc[test,fingerprint],df_query.loc[query,fingerprint]))\n",
    "        tani = sorted(tani,reverse=True)\n",
    "        #print(tani[0])\n",
    "        top_tani.append(tani[0])\n",
    "    df_test_search[\"top_tani\"] = top_tani\n",
    "    \n",
    "    df_test_search.sort_values([\"top_tani\"],ascending=False,inplace=True)\n",
    "    df_test_search.reset_index(drop=True, inplace=True)\n",
    "    \n",
    "    #print(df_test.head())\n",
    "    return df_test_search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4c753071",
   "metadata": {},
   "outputs": [],
   "source": [
    "def df_test_search_result_Phfp(df_query, df_test,fingerprint):\n",
    "    df_test_search = df_test.copy()\n",
    "    # For each molecule in the test set, calculate its similarity to query molecules, and only the highest similarity value was considered\n",
    "    top_tani = []\n",
    "    for test in range(df_test.shape[0]):\n",
    "        #print(df_test.loc[test,\"Name\"])\n",
    "        #print(len(df_test.loc[0,fingerprint]))\n",
    "        tani = []\n",
    "        for query in (range(df_query.shape[0])):\n",
    "            #print(df_test.loc[test,fingerprint])\n",
    "            tani.append(rogot_goldberg(df_test.loc[test,fingerprint],df_query.loc[query,fingerprint]))\n",
    "        tani = sorted(tani,reverse=True)\n",
    "        #print(tani[0])\n",
    "        top_tani.append(tani[0])\n",
    "    df_test_search[\"top_tani\"] = top_tani\n",
    "    \n",
    "    df_test_search.sort_values([\"top_tani\"],ascending=False,inplace=True)\n",
    "    df_test_search.reset_index(drop=True, inplace=True)\n",
    "    \n",
    "    #print(df_test.head())\n",
    "    return df_test_search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "356be147",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Enrichment factor of top 10%\n",
    "def EF10(df_test_search, P_total, N_total):\n",
    "    # Count actives of the top5% of the sorted dataset\n",
    "    N_10 = int(N_total*0.1)\n",
    "    df_test_top = df_test_search.head(N_10)\n",
    "    \n",
    "    P_10 = 0 \n",
    "    for label in df_test_top[\"Pharm class label\"]:\n",
    "        if label != 0:\n",
    "            P_10 += 1\n",
    "    \n",
    "    # Calculate EF10\n",
    "    EF10 = (P_10/N_10)/(P_total/N_total)\n",
    "    return EF10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9e26fa32",
   "metadata": {},
   "outputs": [],
   "source": [
    "def EF10_AUC_average(pharm_class,n_iteration,fingerprint):\n",
    "    print(\"Repeat similarity search \",n_iteration,\"times\",\"for Pharm class\", pharm_class)\n",
    "    EF10_total, AUC_total,optimal_thres = [],[],[]\n",
    "    for iteration in range(n_iteration):\n",
    "        df_query, df_test, P_total, N_total = merge_query_test(pharm_class)\n",
    "        df_test_search = df_test_search_result(df_query, df_test, fingerprint)\n",
    "        \n",
    "        \n",
    "        EF10_score = EF10(df_test_search, P_total, N_total)\n",
    "        EF10_total.append(EF10_score)\n",
    "        \n",
    "        \n",
    "        AUC_score = roc_auc_score(np.array(df_test_search['Pharm class label']), np.array(df_test_search['top_tani']))\n",
    "        AUC_total.append(AUC_score)\n",
    "        \n",
    "        fpr, tpr, thres = roc_curve(np.array(df_test_search['Pharm class label']), np.array(df_test_search['top_tani']), pos_label=pharm_class)\n",
    "        thresOpt = get_opt_thres(fpr, tpr, thres)\n",
    "        optimal_thres.append(thresOpt)\n",
    "        \n",
    "    print(\"Average EF10: %.2f\" %(mean(EF10_total)), \", Average AUC: %0.2f\" %(mean(AUC_total)), \", Average opt thres: %0.2f\" %mean(optimal_thres))\n",
    "    print(\"Std EF10: %.2f\" %np.std(EF10_total), \", Std AUC: %.2f\" %np.std(AUC_total), \", Std opt thres: %0.2f\" %np.std(optimal_thres))\n",
    "    return EF10_total, AUC_total, optimal_thres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "28f7a470",
   "metadata": {},
   "outputs": [],
   "source": [
    "def EF10_AUC_average_Phfp(pharm_class,n_iteration,fingerprint):\n",
    "    print(\"Repeat similarity search \",n_iteration,\"times\",\"for Pharm class\", pharm_class)\n",
    "    EF10_total, AUC_total,optimal_thres = [],[],[]\n",
    "    for iteration in range(n_iteration):\n",
    "        df_query, df_test, P_total, N_total = merge_query_test(pharm_class)\n",
    "        df_test_search = df_test_search_result_Phfp(df_query, df_test, fingerprint)\n",
    "        \n",
    "        \n",
    "        EF10_score = EF10(df_test_search, P_total, N_total)\n",
    "        EF10_total.append(EF10_score)\n",
    "        \n",
    "        \n",
    "        AUC_score = roc_auc_score(np.array(df_test_search['Pharm class label']), np.array(df_test_search['top_tani']))\n",
    "        AUC_total.append(AUC_score)\n",
    "        \n",
    "        fpr, tpr, thres = roc_curve(np.array(df_test_search['Pharm class label']), np.array(df_test_search['top_tani']), pos_label=pharm_class)\n",
    "        thresOpt = get_opt_thres(fpr, tpr, thres)\n",
    "        optimal_thres.append(thresOpt)\n",
    "        \n",
    "    print(\"Average EF10: %.2f\" %(mean(EF10_total)), \", Average AUC: %0.2f\" %(mean(AUC_total)), \", Average opt thres: %0.2f\" %mean(optimal_thres))\n",
    "    print(\"Std EF10: %.2f\" %np.std(EF10_total), \", Std AUC: %.2f\" %np.std(AUC_total), \", Std opt thres: %0.2f\" %np.std(optimal_thres))\n",
    "    return EF10_total, AUC_total, optimal_thres"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8cdef346",
   "metadata": {},
   "source": [
    "## Evaluate similarity search using moleculer fingerprints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "c3a1b6c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_result = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "1ee8d5fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Repeat similarity search for all four pharm classes\n",
    "Pharmclass = \"Serotonergic psychedelics\"\n",
    "Pharmclass_label = 4\n",
    "\n",
    "fingerprint1 = \"RF_p5_maccs\"\n",
    "fingerprint2 = \"RF_p5_morgan\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "5ed0c86e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search using RF_p5_maccs Tanimoto similarity score\n",
      "Repeat similarity search  50 times for Pharm class 4\n",
      "Average EF10: 8.84 , Average AUC: 0.97 , Average opt thres: 0.49\n",
      "Std EF10: 0.58 , Std AUC: 0.01 , Std opt thres: 0.05\n",
      "********************\n",
      "Search using RF_p5_morgan Tanimoto similarity score\n",
      "Repeat similarity search  50 times for Pharm class 4\n",
      "Average EF10: 3.02 , Average AUC: 0.91 , Average opt thres: 0.36\n",
      "Std EF10: 1.41 , Std AUC: 0.01 , Std opt thres: 0.08\n"
     ]
    }
   ],
   "source": [
    "print('Search using', fingerprint1, \"Tanimoto similarity score\")\n",
    "df_result[Pharmclass+'_'+fingerprint1+'_'+\"EF10\"], df_result[Pharmclass+'_'+fingerprint1+'_'+\"AUC\"],df_result[Pharmclass+'_'+fingerprint1+'_'+\"thres\"] = EF10_AUC_average(Pharmclass_label,50,fingerprint1)\n",
    "print(\"**\"*10)\n",
    "print('Search using', fingerprint2, \"Tanimoto similarity score\")\n",
    "df_result[Pharmclass+'_'+fingerprint2+'_'+\"EF10\"], df_result[Pharmclass+'_'+fingerprint2+'_'+\"AUC\"] , df_result[Pharmclass+'_'+fingerprint2+'_'+\"thres\"] = EF10_AUC_average(Pharmclass_label,50,fingerprint2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f08287bc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Depressants_Benzodiazepines_maccsfp_EF10</th>\n",
       "      <th>Depressants_Benzodiazepines_maccsfp_AUC</th>\n",
       "      <th>Depressants_Benzodiazepines_maccsfp_thres</th>\n",
       "      <th>Depressants_Benzodiazepines_morganfp_EF10</th>\n",
       "      <th>Depressants_Benzodiazepines_morganfp_AUC</th>\n",
       "      <th>Depressants_Benzodiazepines_morganfp_thres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>11.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.9070</td>\n",
       "      <td>7.333333</td>\n",
       "      <td>0.988889</td>\n",
       "      <td>0.4286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.666667</td>\n",
       "      <td>0.877778</td>\n",
       "      <td>0.6719</td>\n",
       "      <td>7.333333</td>\n",
       "      <td>0.977778</td>\n",
       "      <td>0.4688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7.333333</td>\n",
       "      <td>0.944444</td>\n",
       "      <td>0.7500</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.4242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7.333333</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>0.9535</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.5323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.8125</td>\n",
       "      <td>7.333333</td>\n",
       "      <td>0.988889</td>\n",
       "      <td>0.4151</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Depressants_Benzodiazepines_maccsfp_EF10  \\\n",
       "0                                 11.000000   \n",
       "1                                  3.666667   \n",
       "2                                  7.333333   \n",
       "3                                  7.333333   \n",
       "4                                 11.000000   \n",
       "\n",
       "   Depressants_Benzodiazepines_maccsfp_AUC  \\\n",
       "0                                 1.000000   \n",
       "1                                 0.877778   \n",
       "2                                 0.944444   \n",
       "3                                 0.888889   \n",
       "4                                 1.000000   \n",
       "\n",
       "   Depressants_Benzodiazepines_maccsfp_thres  \\\n",
       "0                                     0.9070   \n",
       "1                                     0.6719   \n",
       "2                                     0.7500   \n",
       "3                                     0.9535   \n",
       "4                                     0.8125   \n",
       "\n",
       "   Depressants_Benzodiazepines_morganfp_EF10  \\\n",
       "0                                   7.333333   \n",
       "1                                   7.333333   \n",
       "2                                  11.000000   \n",
       "3                                  11.000000   \n",
       "4                                   7.333333   \n",
       "\n",
       "   Depressants_Benzodiazepines_morganfp_AUC  \\\n",
       "0                                  0.988889   \n",
       "1                                  0.977778   \n",
       "2                                  1.000000   \n",
       "3                                  1.000000   \n",
       "4                                  0.988889   \n",
       "\n",
       "   Depressants_Benzodiazepines_morganfp_thres  \n",
       "0                                      0.4286  \n",
       "1                                      0.4688  \n",
       "2                                      0.4242  \n",
       "3                                      0.5323  \n",
       "4                                      0.4151  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_result.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "2bde81ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_result.to_csv(path+\"Structural similarity search Tanimoto EF10 and AUC.csv\",index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48393513",
   "metadata": {},
   "source": [
    "## Evaluate similarity search using pharmacological fingerprints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "4fc9a746",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_result = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "084ffe87",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Repeat similarity search for all four pharm classes\n",
    "Pharmclass = \"Cannabinoids\"\n",
    "Pharmclass_label = 3\n",
    "\n",
    "fingerprint3 = \"RF_p5_morgan\"\n",
    "fingerprint4 = \"RF_p6_morgan\"\n",
    "fingerprint5 = \"RF_p7_morgan\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c95faf63",
   "metadata": {},
   "source": [
    "##### Use Rogot-Goldberg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "a1af270c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search using RF_p5_morgan Rogot-Goldberg similarity score\n",
      "Repeat similarity search  50 times for Pharm class 3\n",
      "Average EF10: 3.29 , Average AUC: 0.92 , Average opt thres: 0.77\n",
      "Std EF10: 1.99 , Std AUC: 0.02 , Std opt thres: 0.06\n",
      "********************\n",
      "Search using RF_p6_morgan Rogot-Goldberg similarity score\n",
      "Repeat similarity search  50 times for Pharm class 3\n",
      "Average EF10: 7.88 , Average AUC: 0.97 , Average opt thres: 0.80\n",
      "Std EF10: 0.97 , Std AUC: 0.01 , Std opt thres: 0.05\n",
      "********************\n",
      "Search using RF_p7_morgan Rogot-Goldberg similarity score\n",
      "Repeat similarity search  50 times for Pharm class 3\n",
      "Average EF10: 3.69 , Average AUC: 0.93 , Average opt thres: 0.79\n",
      "Std EF10: 1.74 , Std AUC: 0.02 , Std opt thres: 0.04\n",
      "********************\n"
     ]
    }
   ],
   "source": [
    "print('Search using', fingerprint3, \"Rogot-Goldberg similarity score\")\n",
    "df_result[Pharmclass+'_'+fingerprint3+'_'+\"EF10\"], df_result[Pharmclass+'_'+fingerprint3+'_'+\"AUC\"],df_result[Pharmclass+'_'+fingerprint3+'_'+\"thres\"] = EF10_AUC_average_Phfp(Pharmclass_label,50,fingerprint3)\n",
    "print(\"**\"*10)\n",
    "print('Search using', fingerprint4, \"Rogot-Goldberg similarity score\")\n",
    "df_result[Pharmclass+'_'+fingerprint4+'_'+\"EF10\"], df_result[Pharmclass+'_'+fingerprint4+'_'+\"AUC\"],df_result[Pharmclass+'_'+fingerprint4+'_'+\"thres\"] = EF10_AUC_average_Phfp(Pharmclass_label,50,fingerprint4)\n",
    "print(\"**\"*10)\n",
    "print('Search using', fingerprint5, \"Rogot-Goldberg similarity score\")\n",
    "df_result[Pharmclass+'_'+fingerprint5+'_'+\"EF10\"], df_result[Pharmclass+'_'+fingerprint5+'_'+\"AUC\"],df_result[Pharmclass+'_'+fingerprint5+'_'+\"thres\"] = EF10_AUC_average_Phfp(Pharmclass_label,50,fingerprint5)\n",
    "print(\"**\"*10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "de70f57c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Stimulants_RF_p5_maccs_EF10</th>\n",
       "      <th>Stimulants_RF_p5_maccs_AUC</th>\n",
       "      <th>Stimulants_RF_p5_maccs_thres</th>\n",
       "      <th>Stimulants_RF_p6_maccs_EF10</th>\n",
       "      <th>Stimulants_RF_p6_maccs_AUC</th>\n",
       "      <th>Stimulants_RF_p6_maccs_thres</th>\n",
       "      <th>Stimulants_RF_p7_maccs_EF10</th>\n",
       "      <th>Stimulants_RF_p7_maccs_AUC</th>\n",
       "      <th>Stimulants_RF_p7_maccs_thres</th>\n",
       "      <th>Cannabinoids_RF_p5_maccs_EF10</th>\n",
       "      <th>...</th>\n",
       "      <th>Depressants_Opioids_RF_p7_maccs_thres</th>\n",
       "      <th>Depressants_Benzodiazepines_RF_p5_maccs_EF10</th>\n",
       "      <th>Depressants_Benzodiazepines_RF_p5_maccs_AUC</th>\n",
       "      <th>Depressants_Benzodiazepines_RF_p5_maccs_thres</th>\n",
       "      <th>Depressants_Benzodiazepines_RF_p6_maccs_EF10</th>\n",
       "      <th>Depressants_Benzodiazepines_RF_p6_maccs_AUC</th>\n",
       "      <th>Depressants_Benzodiazepines_RF_p6_maccs_thres</th>\n",
       "      <th>Depressants_Benzodiazepines_RF_p7_maccs_EF10</th>\n",
       "      <th>Depressants_Benzodiazepines_RF_p7_maccs_AUC</th>\n",
       "      <th>Depressants_Benzodiazepines_RF_p7_maccs_thres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8.768116</td>\n",
       "      <td>0.991736</td>\n",
       "      <td>0.7838</td>\n",
       "      <td>7.971014</td>\n",
       "      <td>0.970723</td>\n",
       "      <td>0.7418</td>\n",
       "      <td>5.739130</td>\n",
       "      <td>0.860015</td>\n",
       "      <td>0.6581</td>\n",
       "      <td>8.25</td>\n",
       "      <td>...</td>\n",
       "      <td>0.8218</td>\n",
       "      <td>7.333333</td>\n",
       "      <td>0.977778</td>\n",
       "      <td>0.9263</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.831</td>\n",
       "      <td>7.333333</td>\n",
       "      <td>0.816667</td>\n",
       "      <td>1.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9.565217</td>\n",
       "      <td>0.991358</td>\n",
       "      <td>0.7831</td>\n",
       "      <td>8.130435</td>\n",
       "      <td>0.968191</td>\n",
       "      <td>0.5988</td>\n",
       "      <td>6.536232</td>\n",
       "      <td>0.856286</td>\n",
       "      <td>0.6566</td>\n",
       "      <td>8.25</td>\n",
       "      <td>...</td>\n",
       "      <td>0.7058</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.8567</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.831</td>\n",
       "      <td>7.333333</td>\n",
       "      <td>0.938889</td>\n",
       "      <td>0.6925</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 45 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Stimulants_RF_p5_maccs_EF10  Stimulants_RF_p5_maccs_AUC  \\\n",
       "0                     8.768116                    0.991736   \n",
       "1                     9.565217                    0.991358   \n",
       "\n",
       "   Stimulants_RF_p5_maccs_thres  Stimulants_RF_p6_maccs_EF10  \\\n",
       "0                        0.7838                     7.971014   \n",
       "1                        0.7831                     8.130435   \n",
       "\n",
       "   Stimulants_RF_p6_maccs_AUC  Stimulants_RF_p6_maccs_thres  \\\n",
       "0                    0.970723                        0.7418   \n",
       "1                    0.968191                        0.5988   \n",
       "\n",
       "   Stimulants_RF_p7_maccs_EF10  Stimulants_RF_p7_maccs_AUC  \\\n",
       "0                     5.739130                    0.860015   \n",
       "1                     6.536232                    0.856286   \n",
       "\n",
       "   Stimulants_RF_p7_maccs_thres  Cannabinoids_RF_p5_maccs_EF10  ...  \\\n",
       "0                        0.6581                           8.25  ...   \n",
       "1                        0.6566                           8.25  ...   \n",
       "\n",
       "   Depressants_Opioids_RF_p7_maccs_thres  \\\n",
       "0                                 0.8218   \n",
       "1                                 0.7058   \n",
       "\n",
       "   Depressants_Benzodiazepines_RF_p5_maccs_EF10  \\\n",
       "0                                      7.333333   \n",
       "1                                     11.000000   \n",
       "\n",
       "   Depressants_Benzodiazepines_RF_p5_maccs_AUC  \\\n",
       "0                                     0.977778   \n",
       "1                                     1.000000   \n",
       "\n",
       "   Depressants_Benzodiazepines_RF_p5_maccs_thres  \\\n",
       "0                                         0.9263   \n",
       "1                                         0.8567   \n",
       "\n",
       "   Depressants_Benzodiazepines_RF_p6_maccs_EF10  \\\n",
       "0                                          11.0   \n",
       "1                                          11.0   \n",
       "\n",
       "   Depressants_Benzodiazepines_RF_p6_maccs_AUC  \\\n",
       "0                                          1.0   \n",
       "1                                          1.0   \n",
       "\n",
       "   Depressants_Benzodiazepines_RF_p6_maccs_thres  \\\n",
       "0                                          0.831   \n",
       "1                                          0.831   \n",
       "\n",
       "   Depressants_Benzodiazepines_RF_p7_maccs_EF10  \\\n",
       "0                                      7.333333   \n",
       "1                                      7.333333   \n",
       "\n",
       "   Depressants_Benzodiazepines_RF_p7_maccs_AUC  \\\n",
       "0                                     0.816667   \n",
       "1                                     0.938889   \n",
       "\n",
       "   Depressants_Benzodiazepines_RF_p7_maccs_thres  \n",
       "0                                         1.0000  \n",
       "1                                         0.6925  \n",
       "\n",
       "[2 rows x 45 columns]"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_result.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "a830d41a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_result.to_csv(path+\"Ph-fp maccsfp similarity search Rogot-Goldberg EF10 and AUC.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4136bfea",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}