构建基模型.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "15a6c63a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os, re, math, platform\n",
    "from pathlib import Path\n",
    "import matplotlib.pyplot as plt\n",
    "import json\n",
    "import joblib\n",
    "from scipy.stats import randint as sp_randint\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import matthews_corrcoef, confusion_matrix\n",
    "from sklearn.metrics import precision_recall_curve, roc_curve, auc, fbeta_score\n",
    "from imblearn.metrics import geometric_mean_score\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from xgboost import XGBClassifier \n",
    "from xgboost import plot_importance\n",
    "from sklearn.ensemble import GradientBoostingClassifier,RandomForestClassifier,ExtraTreesClassifier,AdaBoostClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.svm import SVC\n",
    "from Bio import SeqIO\n",
    "from Bio.SeqUtils.ProtParam import ProteinAnalysis as PA\n",
    "from modlamp.descriptors import PeptideDescriptor, GlobalDescriptor\n",
    "from matplotlib import pyplot\n",
    "from sklearn.metrics import matthews_corrcoef, confusion_matrix,precision_recall_curve, roc_curve, auc, fbeta_score,roc_auc_score\n",
    "from fea_extract import read_fasta,insert_AAC,insert_DPC,insert_CKSAAGP,insert_CTD,insert_PAAC,insert_AAI,insert_GTPC,insert_QSO,insert_AAE,insert_PSAAC,insert_word2int,insert_ASDC\n",
    "import warnings \n",
    "from tools import cv,evaluate\n",
    "warnings.filterwarnings('ignore')\n",
    "seed = 10\n",
    "from sklearn.preprocessing import MinMaxScaler, StandardScaler \n",
    "Path('./results/predict_proba/').mkdir(exist_ok=True,parents=True)\n",
    "Path('./results/evaluate/').mkdir(exist_ok=True,parents=True)\n",
    "Path('./results/model/').mkdir(exist_ok=True,parents=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "18df24a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "GBDT = GradientBoostingClassifier(random_state=seed)\n",
    "ET = ExtraTreesClassifier(random_state=seed)\n",
    "SVM = SVC(random_state=seed,probability=True)\n",
    "MLP = MLPClassifier(hidden_layer_sizes=64,learning_rate=\"adaptive\",random_state=seed)\n",
    "RF = RandomForestClassifier(random_state=seed)\n",
    "XGBoost = XGBClassifier(random_state=seed)\n",
    "LR  = LogisticRegression(random_state=seed)\n",
    "AB = AdaBoostClassifier(random_state=seed)\n",
    "clf_feature_order = {\n",
    "#     \"GA\": [\"XGBoost\",\"RF\",\"ET\",\"GBDT\"],\n",
    "    \"Boruta\": [\"XGBoost\",\"RF\",\"ET\",\"GBDT\"],\n",
    "#     \"Fscore\": [\"XGBoost\",\"RF\",\"ET\",\"GBDT\"],\n",
    "#     \"MI\": [\"XGBoost\",\"RF\",\"ET\",\"GBDT\"],\n",
    "    \"MRMD\": [\"XGBoost\",\"RF\",\"ET\",\"GBDT\"],\n",
    "    \"XGB\": [\"XGBoost\",\"RF\",\"ET\",\"GBDT\"],\n",
    "#     \"RFE\": [\"XGBoost\",\"RF\",\"ET\",\"GBDT\"],\n",
    "}\n",
    "ml = [\"XGBoost\",\"RF\",\"ET\",\"GBDT\",'AB']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "36f07832",
   "metadata": {},
   "outputs": [],
   "source": [
    "#定义基学习器\n",
    "def base_clf(clf,X_train,y_train,model_name,n_folds=5):\n",
    "    ntrain = X_train.shape[0] #训练样本个数。\n",
    "    nclass = len(np.unique(y_train)) #类别个数。\n",
    "    kf = KFold(n_splits=n_folds, shuffle=True, random_state=seed)#K-折叠交叉验证。\n",
    "    base_pf_train = np.zeros((ntrain,nclass)) #返回给定形状和类型的新数组，用零填充。\n",
    "    base_cf_train = np.zeros((ntrain))\n",
    "    \n",
    "    for train_index, test_index in kf.split(X_train,y_train):\n",
    "        kf_X_train,kf_y_train = X_train[train_index],y_train[train_index]\n",
    "        kf_X_test = X_train[test_index]\n",
    "\n",
    "        clf.fit(kf_X_train, kf_y_train)\n",
    "        base_pf_train[test_index] = clf.predict_proba(kf_X_test)\n",
    "        base_cf_train[test_index] = clf.predict(kf_X_test)\n",
    "    clf.fit(X_train,y_train)\n",
    "    joblib.dump(model,f'./results/model/{model_name}')\n",
    "    return base_pf_train[:,-1], base_cf_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "cb42d88c",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Boruta_XGBoost\n",
      "Boruta_RF\n",
      "Boruta_ET\n",
      "Boruta_GBDT\n",
      "MRMD_XGBoost\n",
      "MRMD_RF\n",
      "MRMD_ET\n",
      "MRMD_GBDT\n",
      "XGB_XGBoost\n",
      "XGB_RF\n",
      "XGB_ET\n",
      "XGB_GBDT\n"
     ]
    }
   ],
   "source": [
    "X_train_base_pf = []\n",
    "X_test_base_pf = []\n",
    "X_train_base_cf = []\n",
    "X_test_base_cf = []\n",
    "X_train_base_pcf = []\n",
    "X_test_base_pcf = []\n",
    "\n",
    "all_eval_dic = []\n",
    "Index = []\n",
    "index = []\n",
    "ALL_evals_train = pd.DataFrame()\n",
    "for (n,m) in clf_feature_order.items():\n",
    "    X_train = pd.read_csv('Feature_different/X_train_{}.csv'.format(n)).to_numpy()\n",
    "    X_test = pd.read_csv('Feature_different/X_test_{}.csv'.format(n)).to_numpy()\n",
    "    y_train = pd.read_csv('data/train/y_train_resampled.csv').to_numpy()\n",
    "    y_test = pd.read_csv('data/test/y_test.csv').to_numpy()\n",
    "#     np.random.seed(1234)\n",
    "#     idx = np.random.permutation(len(X_train))\n",
    "#     X_train = X_train[idx]\n",
    "#     y_train = y_train[idx]\n",
    "\n",
    "    for c in m:\n",
    "        model = eval(c)\n",
    "        print(\"{}_{:s}\".format(n,c))\n",
    "        X_train_pf,X_train_cf = base_clf(model,X_train,y_train,f'{n}_{c}.m')\n",
    "        Evals = cv(model,X_train,y_train)\n",
    "        ALL_evals_train = pd.concat([ALL_evals_train,Evals],axis=1)\n",
    "        index.append('{}_{}'.format(n,c))\n",
    "                # PF\n",
    "        X_train_base_pf.append(X_train_pf) \n",
    "        X_train_pf_dic = np.array(X_train_base_pf).T\n",
    "             # CF\n",
    "        X_train_base_cf.append(X_train_cf) \n",
    "        X_train_cf_dic = np.array(X_train_base_cf).T\n",
    "        # PCF\n",
    "        X_train_base_pcf.append(X_train_pf)\n",
    "        X_train_base_pcf.append(X_train_cf)\n",
    "        X_train_pcf_dic = np.array(X_train_base_pcf).T\n",
    "\n",
    "        base_model = joblib.load('results/model/{}_{}.m'.format(n,c))\n",
    "                # PF\n",
    "        X_test_pf = base_model.predict_proba(X_test)[:,-1]\n",
    "        X_test_base_pf.append(X_test_pf) \n",
    "        X_test_pf_dic = np.array(X_test_base_pf).T\n",
    "                # CF\n",
    "        X_test_cf = base_model.predict(X_test)\n",
    "        X_test_base_cf.append(X_test_cf) \n",
    "        X_test_cf_dic = np.array(X_test_base_cf).T\n",
    "                # PCF\n",
    "        X_test_base_pcf.append(X_test_pf)\n",
    "        X_test_base_pcf.append(X_test_cf)\n",
    "        X_test_pcf_dic = np.array(X_test_base_pcf).T\n",
    "        Index.append(\"{}_{}\".format(n,c))\n",
    "\n",
    "        eval_dic = evaluate(X_test,y_test,base_model)\n",
    "#     print(\"test：\",eval_dic)\n",
    "        all_eval_dic = all_eval_dic+[eval_dic]\n",
    "    \n",
    "all_eval_dic = pd.DataFrame(all_eval_dic,index=Index) #\n",
    "all_eval_dic.to_csv(os.path.join(\"results/evaluate/tow_evalu.csv\"))\n",
    "# all_eval_dic.plot.bar(figsize=(20,10),fontsize=15,rot=45,grid = False, width = 0.7,title=\"{:s}\".format(m))\n",
    "\n",
    "#输出概率向量  \n",
    "#训练集\n",
    "X_train_pf_dic = pd.DataFrame(X_train_pf_dic)\n",
    "X_train_pf_dic.to_csv(os.path.join(\"results/predict_proba/X_train_pf.csv\"),index=False)\n",
    "X_train_cf_dic = pd.DataFrame(X_train_cf_dic)\n",
    "X_train_cf_dic.to_csv(os.path.join(\"results/predict_proba/X_train_cf.csv\"),index=False)\n",
    "X_train_pcf_dic = pd.DataFrame(X_train_pcf_dic)\n",
    "X_train_pcf_dic.to_csv(os.path.join(\"results/predict_proba/X_train_pcf.csv\"),index=False)\n",
    "y_train = pd.DataFrame(y_train)\n",
    "y_train.to_csv(os.path.join(\"results/y_train.csv\"),index=False)\n",
    "#测试集\n",
    "X_test_cf_dic = pd.DataFrame(X_test_cf_dic)\n",
    "X_test_cf_dic.to_csv(os.path.join(\"results/predict_proba/X_test_cf.csv\"),index=False)\n",
    "X_test_pf_dic = pd.DataFrame(X_test_pf_dic)\n",
    "X_test_pf_dic.to_csv(os.path.join(\"results/predict_proba/X_test_pf.csv\"),index=False)\n",
    "X_test_pcf_dic = pd.DataFrame(X_test_pcf_dic)\n",
    "X_test_pcf_dic.to_csv(os.path.join(\"results/predict_proba/X_test_pcf.csv\"),index=False)\n",
    "y_test = pd.DataFrame(y_test)\n",
    "y_test.to_csv(os.path.join(\"results/y_test.csv\"),index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "89b18919",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ACC</th>\n",
       "      <td>0.982085</td>\n",
       "      <td>0.984575</td>\n",
       "      <td>0.993032</td>\n",
       "      <td>0.975613</td>\n",
       "      <td>0.985566</td>\n",
       "      <td>0.988053</td>\n",
       "      <td>0.998010</td>\n",
       "      <td>0.983077</td>\n",
       "      <td>0.986560</td>\n",
       "      <td>0.985066</td>\n",
       "      <td>0.998506</td>\n",
       "      <td>0.984073</td>\n",
       "      <td>0.986558</td>\n",
       "      <td>0.990546</td>\n",
       "      <td>0.998007</td>\n",
       "      <td>0.984571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1</th>\n",
       "      <td>0.985568</td>\n",
       "      <td>0.987730</td>\n",
       "      <td>0.994535</td>\n",
       "      <td>0.980507</td>\n",
       "      <td>0.988404</td>\n",
       "      <td>0.990393</td>\n",
       "      <td>0.998416</td>\n",
       "      <td>0.986516</td>\n",
       "      <td>0.989179</td>\n",
       "      <td>0.988003</td>\n",
       "      <td>0.998815</td>\n",
       "      <td>0.987229</td>\n",
       "      <td>0.989220</td>\n",
       "      <td>0.992372</td>\n",
       "      <td>0.998424</td>\n",
       "      <td>0.987762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F2</th>\n",
       "      <td>0.984522</td>\n",
       "      <td>0.985914</td>\n",
       "      <td>0.995433</td>\n",
       "      <td>0.982525</td>\n",
       "      <td>0.988082</td>\n",
       "      <td>0.988883</td>\n",
       "      <td>0.998402</td>\n",
       "      <td>0.986856</td>\n",
       "      <td>0.987944</td>\n",
       "      <td>0.987921</td>\n",
       "      <td>0.999525</td>\n",
       "      <td>0.987615</td>\n",
       "      <td>0.988405</td>\n",
       "      <td>0.990164</td>\n",
       "      <td>0.998890</td>\n",
       "      <td>0.988317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GMean</th>\n",
       "      <td>0.980922</td>\n",
       "      <td>0.983666</td>\n",
       "      <td>0.991673</td>\n",
       "      <td>0.972199</td>\n",
       "      <td>0.984339</td>\n",
       "      <td>0.987692</td>\n",
       "      <td>0.997799</td>\n",
       "      <td>0.981058</td>\n",
       "      <td>0.986077</td>\n",
       "      <td>0.983710</td>\n",
       "      <td>0.997957</td>\n",
       "      <td>0.982323</td>\n",
       "      <td>0.985619</td>\n",
       "      <td>0.990821</td>\n",
       "      <td>0.997536</td>\n",
       "      <td>0.982381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SEN</th>\n",
       "      <td>0.983845</td>\n",
       "      <td>0.984729</td>\n",
       "      <td>0.996039</td>\n",
       "      <td>0.983890</td>\n",
       "      <td>0.987884</td>\n",
       "      <td>0.987897</td>\n",
       "      <td>0.998393</td>\n",
       "      <td>0.987097</td>\n",
       "      <td>0.987133</td>\n",
       "      <td>0.987881</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.987881</td>\n",
       "      <td>0.987881</td>\n",
       "      <td>0.988710</td>\n",
       "      <td>0.999203</td>\n",
       "      <td>0.988707</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PREC</th>\n",
       "      <td>0.987394</td>\n",
       "      <td>0.990856</td>\n",
       "      <td>0.993068</td>\n",
       "      <td>0.977217</td>\n",
       "      <td>0.989007</td>\n",
       "      <td>0.992997</td>\n",
       "      <td>0.998444</td>\n",
       "      <td>0.986003</td>\n",
       "      <td>0.991290</td>\n",
       "      <td>0.988201</td>\n",
       "      <td>0.997635</td>\n",
       "      <td>0.986618</td>\n",
       "      <td>0.990655</td>\n",
       "      <td>0.996125</td>\n",
       "      <td>0.997654</td>\n",
       "      <td>0.986912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SPEC</th>\n",
       "      <td>0.978092</td>\n",
       "      <td>0.982751</td>\n",
       "      <td>0.987367</td>\n",
       "      <td>0.960735</td>\n",
       "      <td>0.980881</td>\n",
       "      <td>0.987574</td>\n",
       "      <td>0.997212</td>\n",
       "      <td>0.975149</td>\n",
       "      <td>0.985068</td>\n",
       "      <td>0.979631</td>\n",
       "      <td>0.995921</td>\n",
       "      <td>0.976842</td>\n",
       "      <td>0.983455</td>\n",
       "      <td>0.992984</td>\n",
       "      <td>0.995881</td>\n",
       "      <td>0.976225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MCC</th>\n",
       "      <td>0.961917</td>\n",
       "      <td>0.966895</td>\n",
       "      <td>0.984878</td>\n",
       "      <td>0.947789</td>\n",
       "      <td>0.969269</td>\n",
       "      <td>0.974647</td>\n",
       "      <td>0.995725</td>\n",
       "      <td>0.963711</td>\n",
       "      <td>0.971392</td>\n",
       "      <td>0.968192</td>\n",
       "      <td>0.996777</td>\n",
       "      <td>0.965955</td>\n",
       "      <td>0.971369</td>\n",
       "      <td>0.979988</td>\n",
       "      <td>0.995710</td>\n",
       "      <td>0.966828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AUC</th>\n",
       "      <td>0.997754</td>\n",
       "      <td>0.998690</td>\n",
       "      <td>0.999650</td>\n",
       "      <td>0.997667</td>\n",
       "      <td>0.998528</td>\n",
       "      <td>0.998955</td>\n",
       "      <td>0.999978</td>\n",
       "      <td>0.998280</td>\n",
       "      <td>0.998321</td>\n",
       "      <td>0.999249</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.998342</td>\n",
       "      <td>0.998775</td>\n",
       "      <td>0.999582</td>\n",
       "      <td>0.999987</td>\n",
       "      <td>0.998608</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              0         0         0         0         0         0         0  \\\n",
       "ACC    0.982085  0.984575  0.993032  0.975613  0.985566  0.988053  0.998010   \n",
       "F1     0.985568  0.987730  0.994535  0.980507  0.988404  0.990393  0.998416   \n",
       "F2     0.984522  0.985914  0.995433  0.982525  0.988082  0.988883  0.998402   \n",
       "GMean  0.980922  0.983666  0.991673  0.972199  0.984339  0.987692  0.997799   \n",
       "SEN    0.983845  0.984729  0.996039  0.983890  0.987884  0.987897  0.998393   \n",
       "PREC   0.987394  0.990856  0.993068  0.977217  0.989007  0.992997  0.998444   \n",
       "SPEC   0.978092  0.982751  0.987367  0.960735  0.980881  0.987574  0.997212   \n",
       "MCC    0.961917  0.966895  0.984878  0.947789  0.969269  0.974647  0.995725   \n",
       "AUC    0.997754  0.998690  0.999650  0.997667  0.998528  0.998955  0.999978   \n",
       "\n",
       "              0         0         0         0         0         0         0  \\\n",
       "ACC    0.983077  0.986560  0.985066  0.998506  0.984073  0.986558  0.990546   \n",
       "F1     0.986516  0.989179  0.988003  0.998815  0.987229  0.989220  0.992372   \n",
       "F2     0.986856  0.987944  0.987921  0.999525  0.987615  0.988405  0.990164   \n",
       "GMean  0.981058  0.986077  0.983710  0.997957  0.982323  0.985619  0.990821   \n",
       "SEN    0.987097  0.987133  0.987881  1.000000  0.987881  0.987881  0.988710   \n",
       "PREC   0.986003  0.991290  0.988201  0.997635  0.986618  0.990655  0.996125   \n",
       "SPEC   0.975149  0.985068  0.979631  0.995921  0.976842  0.983455  0.992984   \n",
       "MCC    0.963711  0.971392  0.968192  0.996777  0.965955  0.971369  0.979988   \n",
       "AUC    0.998280  0.998321  0.999249  1.000000  0.998342  0.998775  0.999582   \n",
       "\n",
       "              0         0  \n",
       "ACC    0.998007  0.984571  \n",
       "F1     0.998424  0.987762  \n",
       "F2     0.998890  0.988317  \n",
       "GMean  0.997536  0.982381  \n",
       "SEN    0.999203  0.988707  \n",
       "PREC   0.997654  0.986912  \n",
       "SPEC   0.995881  0.976225  \n",
       "MCC    0.995710  0.966828  \n",
       "AUC    0.999987  0.998608  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ALL_evals_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ae929e62",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ACC</th>\n",
       "      <th>F1</th>\n",
       "      <th>F2</th>\n",
       "      <th>GMean</th>\n",
       "      <th>SEN</th>\n",
       "      <th>PREC</th>\n",
       "      <th>SPEC</th>\n",
       "      <th>MCC</th>\n",
       "      <th>AUC</th>\n",
       "      <th>AUPR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Boruta_XGBoost</th>\n",
       "      <td>0.895105</td>\n",
       "      <td>0.563107</td>\n",
       "      <td>0.728643</td>\n",
       "      <td>0.900208</td>\n",
       "      <td>0.90625</td>\n",
       "      <td>0.408451</td>\n",
       "      <td>0.894207</td>\n",
       "      <td>0.565897</td>\n",
       "      <td>0.946001</td>\n",
       "      <td>0.742464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Boruta_RF</th>\n",
       "      <td>0.906760</td>\n",
       "      <td>0.574468</td>\n",
       "      <td>0.710526</td>\n",
       "      <td>0.877134</td>\n",
       "      <td>0.84375</td>\n",
       "      <td>0.435484</td>\n",
       "      <td>0.911839</td>\n",
       "      <td>0.564582</td>\n",
       "      <td>0.939822</td>\n",
       "      <td>0.653450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Boruta_ET</th>\n",
       "      <td>0.902098</td>\n",
       "      <td>0.580000</td>\n",
       "      <td>0.739796</td>\n",
       "      <td>0.904004</td>\n",
       "      <td>0.90625</td>\n",
       "      <td>0.426471</td>\n",
       "      <td>0.901763</td>\n",
       "      <td>0.581274</td>\n",
       "      <td>0.945686</td>\n",
       "      <td>0.678939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Boruta_GBDT</th>\n",
       "      <td>0.888112</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.735294</td>\n",
       "      <td>0.910425</td>\n",
       "      <td>0.93750</td>\n",
       "      <td>0.394737</td>\n",
       "      <td>0.884131</td>\n",
       "      <td>0.565396</td>\n",
       "      <td>0.939310</td>\n",
       "      <td>0.554444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MRMD_XGBoost</th>\n",
       "      <td>0.876457</td>\n",
       "      <td>0.513761</td>\n",
       "      <td>0.682927</td>\n",
       "      <td>0.875787</td>\n",
       "      <td>0.87500</td>\n",
       "      <td>0.363636</td>\n",
       "      <td>0.876574</td>\n",
       "      <td>0.514548</td>\n",
       "      <td>0.945529</td>\n",
       "      <td>0.719694</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MRMD_RF</th>\n",
       "      <td>0.890443</td>\n",
       "      <td>0.543689</td>\n",
       "      <td>0.703518</td>\n",
       "      <td>0.883304</td>\n",
       "      <td>0.87500</td>\n",
       "      <td>0.394366</td>\n",
       "      <td>0.891688</td>\n",
       "      <td>0.542023</td>\n",
       "      <td>0.938917</td>\n",
       "      <td>0.652368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MRMD_ET</th>\n",
       "      <td>0.890443</td>\n",
       "      <td>0.525253</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.853574</td>\n",
       "      <td>0.81250</td>\n",
       "      <td>0.388060</td>\n",
       "      <td>0.896725</td>\n",
       "      <td>0.513290</td>\n",
       "      <td>0.941633</td>\n",
       "      <td>0.625011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MRMD_GBDT</th>\n",
       "      <td>0.857809</td>\n",
       "      <td>0.487395</td>\n",
       "      <td>0.674419</td>\n",
       "      <td>0.879688</td>\n",
       "      <td>0.90625</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.853904</td>\n",
       "      <td>0.496706</td>\n",
       "      <td>0.920576</td>\n",
       "      <td>0.642158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XGB_XGBoost</th>\n",
       "      <td>0.885781</td>\n",
       "      <td>0.542056</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.895122</td>\n",
       "      <td>0.90625</td>\n",
       "      <td>0.386667</td>\n",
       "      <td>0.884131</td>\n",
       "      <td>0.546732</td>\n",
       "      <td>0.954503</td>\n",
       "      <td>0.747171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XGB_RF</th>\n",
       "      <td>0.892774</td>\n",
       "      <td>0.549020</td>\n",
       "      <td>0.707071</td>\n",
       "      <td>0.884551</td>\n",
       "      <td>0.87500</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.894207</td>\n",
       "      <td>0.546911</td>\n",
       "      <td>0.941514</td>\n",
       "      <td>0.662093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XGB_ET</th>\n",
       "      <td>0.904429</td>\n",
       "      <td>0.559140</td>\n",
       "      <td>0.687831</td>\n",
       "      <td>0.860737</td>\n",
       "      <td>0.81250</td>\n",
       "      <td>0.426230</td>\n",
       "      <td>0.911839</td>\n",
       "      <td>0.544908</td>\n",
       "      <td>0.940649</td>\n",
       "      <td>0.673215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XGB_GBDT</th>\n",
       "      <td>0.857809</td>\n",
       "      <td>0.478632</td>\n",
       "      <td>0.657277</td>\n",
       "      <td>0.865662</td>\n",
       "      <td>0.87500</td>\n",
       "      <td>0.329412</td>\n",
       "      <td>0.856423</td>\n",
       "      <td>0.482115</td>\n",
       "      <td>0.933407</td>\n",
       "      <td>0.710766</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     ACC        F1        F2     GMean      SEN      PREC  \\\n",
       "Boruta_XGBoost  0.895105  0.563107  0.728643  0.900208  0.90625  0.408451   \n",
       "Boruta_RF       0.906760  0.574468  0.710526  0.877134  0.84375  0.435484   \n",
       "Boruta_ET       0.902098  0.580000  0.739796  0.904004  0.90625  0.426471   \n",
       "Boruta_GBDT     0.888112  0.555556  0.735294  0.910425  0.93750  0.394737   \n",
       "MRMD_XGBoost    0.876457  0.513761  0.682927  0.875787  0.87500  0.363636   \n",
       "MRMD_RF         0.890443  0.543689  0.703518  0.883304  0.87500  0.394366   \n",
       "MRMD_ET         0.890443  0.525253  0.666667  0.853574  0.81250  0.388060   \n",
       "MRMD_GBDT       0.857809  0.487395  0.674419  0.879688  0.90625  0.333333   \n",
       "XGB_XGBoost     0.885781  0.542056  0.714286  0.895122  0.90625  0.386667   \n",
       "XGB_RF          0.892774  0.549020  0.707071  0.884551  0.87500  0.400000   \n",
       "XGB_ET          0.904429  0.559140  0.687831  0.860737  0.81250  0.426230   \n",
       "XGB_GBDT        0.857809  0.478632  0.657277  0.865662  0.87500  0.329412   \n",
       "\n",
       "                    SPEC       MCC       AUC      AUPR  \n",
       "Boruta_XGBoost  0.894207  0.565897  0.946001  0.742464  \n",
       "Boruta_RF       0.911839  0.564582  0.939822  0.653450  \n",
       "Boruta_ET       0.901763  0.581274  0.945686  0.678939  \n",
       "Boruta_GBDT     0.884131  0.565396  0.939310  0.554444  \n",
       "MRMD_XGBoost    0.876574  0.514548  0.945529  0.719694  \n",
       "MRMD_RF         0.891688  0.542023  0.938917  0.652368  \n",
       "MRMD_ET         0.896725  0.513290  0.941633  0.625011  \n",
       "MRMD_GBDT       0.853904  0.496706  0.920576  0.642158  \n",
       "XGB_XGBoost     0.884131  0.546732  0.954503  0.747171  \n",
       "XGB_RF          0.894207  0.546911  0.941514  0.662093  \n",
       "XGB_ET          0.911839  0.544908  0.940649  0.673215  \n",
       "XGB_GBDT        0.856423  0.482115  0.933407  0.710766  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_eval_dic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3e59e78f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9efe671a",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_eval = pd.read_csv('results/evaluate/tow_evalu.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1cba650",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "a4453b25",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ACC</th>\n",
       "      <th>F1</th>\n",
       "      <th>F2</th>\n",
       "      <th>GMean</th>\n",
       "      <th>SEN</th>\n",
       "      <th>PREC</th>\n",
       "      <th>SPEC</th>\n",
       "      <th>MCC</th>\n",
       "      <th>AUC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Boruta_XGBoost</th>\n",
       "      <td>0.895105</td>\n",
       "      <td>0.563107</td>\n",
       "      <td>0.728643</td>\n",
       "      <td>0.900208</td>\n",
       "      <td>0.90625</td>\n",
       "      <td>0.408451</td>\n",
       "      <td>0.894207</td>\n",
       "      <td>0.565897</td>\n",
       "      <td>0.946001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Boruta_RF</th>\n",
       "      <td>0.906760</td>\n",
       "      <td>0.574468</td>\n",
       "      <td>0.710526</td>\n",
       "      <td>0.877134</td>\n",
       "      <td>0.84375</td>\n",
       "      <td>0.435484</td>\n",
       "      <td>0.911839</td>\n",
       "      <td>0.564582</td>\n",
       "      <td>0.939822</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Boruta_ET</th>\n",
       "      <td>0.902098</td>\n",
       "      <td>0.580000</td>\n",
       "      <td>0.739796</td>\n",
       "      <td>0.904004</td>\n",
       "      <td>0.90625</td>\n",
       "      <td>0.426471</td>\n",
       "      <td>0.901763</td>\n",
       "      <td>0.581274</td>\n",
       "      <td>0.945686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Boruta_GBDT</th>\n",
       "      <td>0.888112</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.735294</td>\n",
       "      <td>0.910425</td>\n",
       "      <td>0.93750</td>\n",
       "      <td>0.394737</td>\n",
       "      <td>0.884131</td>\n",
       "      <td>0.565396</td>\n",
       "      <td>0.939310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MI_XGBoost</th>\n",
       "      <td>0.878788</td>\n",
       "      <td>0.527273</td>\n",
       "      <td>0.703883</td>\n",
       "      <td>0.891289</td>\n",
       "      <td>0.90625</td>\n",
       "      <td>0.371795</td>\n",
       "      <td>0.876574</td>\n",
       "      <td>0.533253</td>\n",
       "      <td>0.950252</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MI_RF</th>\n",
       "      <td>0.878788</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.650000</td>\n",
       "      <td>0.847559</td>\n",
       "      <td>0.81250</td>\n",
       "      <td>0.361111</td>\n",
       "      <td>0.884131</td>\n",
       "      <td>0.489748</td>\n",
       "      <td>0.939153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MI_ET</th>\n",
       "      <td>0.902098</td>\n",
       "      <td>0.553191</td>\n",
       "      <td>0.684211</td>\n",
       "      <td>0.859548</td>\n",
       "      <td>0.81250</td>\n",
       "      <td>0.419355</td>\n",
       "      <td>0.909320</td>\n",
       "      <td>0.539349</td>\n",
       "      <td>0.937933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MI_GBDT</th>\n",
       "      <td>0.855478</td>\n",
       "      <td>0.474576</td>\n",
       "      <td>0.654206</td>\n",
       "      <td>0.864388</td>\n",
       "      <td>0.87500</td>\n",
       "      <td>0.325581</td>\n",
       "      <td>0.853904</td>\n",
       "      <td>0.478349</td>\n",
       "      <td>0.941200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MRMD_XGBoost</th>\n",
       "      <td>0.885781</td>\n",
       "      <td>0.524272</td>\n",
       "      <td>0.678392</td>\n",
       "      <td>0.866162</td>\n",
       "      <td>0.84375</td>\n",
       "      <td>0.380282</td>\n",
       "      <td>0.889169</td>\n",
       "      <td>0.518150</td>\n",
       "      <td>0.946159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MRMD_RF</th>\n",
       "      <td>0.899767</td>\n",
       "      <td>0.547368</td>\n",
       "      <td>0.680628</td>\n",
       "      <td>0.858356</td>\n",
       "      <td>0.81250</td>\n",
       "      <td>0.412698</td>\n",
       "      <td>0.906801</td>\n",
       "      <td>0.533913</td>\n",
       "      <td>0.940019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MRMD_ET</th>\n",
       "      <td>0.918415</td>\n",
       "      <td>0.597701</td>\n",
       "      <td>0.710383</td>\n",
       "      <td>0.867841</td>\n",
       "      <td>0.81250</td>\n",
       "      <td>0.472727</td>\n",
       "      <td>0.926952</td>\n",
       "      <td>0.581116</td>\n",
       "      <td>0.942223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MRMD_GBDT</th>\n",
       "      <td>0.871795</td>\n",
       "      <td>0.513274</td>\n",
       "      <td>0.693780</td>\n",
       "      <td>0.887439</td>\n",
       "      <td>0.90625</td>\n",
       "      <td>0.358025</td>\n",
       "      <td>0.869018</td>\n",
       "      <td>0.520463</td>\n",
       "      <td>0.938602</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XGB_XGBoost</th>\n",
       "      <td>0.876457</td>\n",
       "      <td>0.522523</td>\n",
       "      <td>0.700483</td>\n",
       "      <td>0.890007</td>\n",
       "      <td>0.90625</td>\n",
       "      <td>0.367089</td>\n",
       "      <td>0.874055</td>\n",
       "      <td>0.528917</td>\n",
       "      <td>0.953794</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XGB_RF</th>\n",
       "      <td>0.892774</td>\n",
       "      <td>0.530612</td>\n",
       "      <td>0.670103</td>\n",
       "      <td>0.854772</td>\n",
       "      <td>0.81250</td>\n",
       "      <td>0.393939</td>\n",
       "      <td>0.899244</td>\n",
       "      <td>0.518285</td>\n",
       "      <td>0.938090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XGB_ET</th>\n",
       "      <td>0.906760</td>\n",
       "      <td>0.574468</td>\n",
       "      <td>0.710526</td>\n",
       "      <td>0.877134</td>\n",
       "      <td>0.84375</td>\n",
       "      <td>0.435484</td>\n",
       "      <td>0.911839</td>\n",
       "      <td>0.564582</td>\n",
       "      <td>0.946946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XGB_GBDT</th>\n",
       "      <td>0.860140</td>\n",
       "      <td>0.482759</td>\n",
       "      <td>0.660377</td>\n",
       "      <td>0.866934</td>\n",
       "      <td>0.87500</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.858942</td>\n",
       "      <td>0.485940</td>\n",
       "      <td>0.939074</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     ACC        F1        F2     GMean      SEN      PREC  \\\n",
       "Boruta_XGBoost  0.895105  0.563107  0.728643  0.900208  0.90625  0.408451   \n",
       "Boruta_RF       0.906760  0.574468  0.710526  0.877134  0.84375  0.435484   \n",
       "Boruta_ET       0.902098  0.580000  0.739796  0.904004  0.90625  0.426471   \n",
       "Boruta_GBDT     0.888112  0.555556  0.735294  0.910425  0.93750  0.394737   \n",
       "MI_XGBoost      0.878788  0.527273  0.703883  0.891289  0.90625  0.371795   \n",
       "MI_RF           0.878788  0.500000  0.650000  0.847559  0.81250  0.361111   \n",
       "MI_ET           0.902098  0.553191  0.684211  0.859548  0.81250  0.419355   \n",
       "MI_GBDT         0.855478  0.474576  0.654206  0.864388  0.87500  0.325581   \n",
       "MRMD_XGBoost    0.885781  0.524272  0.678392  0.866162  0.84375  0.380282   \n",
       "MRMD_RF         0.899767  0.547368  0.680628  0.858356  0.81250  0.412698   \n",
       "MRMD_ET         0.918415  0.597701  0.710383  0.867841  0.81250  0.472727   \n",
       "MRMD_GBDT       0.871795  0.513274  0.693780  0.887439  0.90625  0.358025   \n",
       "XGB_XGBoost     0.876457  0.522523  0.700483  0.890007  0.90625  0.367089   \n",
       "XGB_RF          0.892774  0.530612  0.670103  0.854772  0.81250  0.393939   \n",
       "XGB_ET          0.906760  0.574468  0.710526  0.877134  0.84375  0.435484   \n",
       "XGB_GBDT        0.860140  0.482759  0.660377  0.866934  0.87500  0.333333   \n",
       "\n",
       "                    SPEC       MCC       AUC  \n",
       "Boruta_XGBoost  0.894207  0.565897  0.946001  \n",
       "Boruta_RF       0.911839  0.564582  0.939822  \n",
       "Boruta_ET       0.901763  0.581274  0.945686  \n",
       "Boruta_GBDT     0.884131  0.565396  0.939310  \n",
       "MI_XGBoost      0.876574  0.533253  0.950252  \n",
       "MI_RF           0.884131  0.489748  0.939153  \n",
       "MI_ET           0.909320  0.539349  0.937933  \n",
       "MI_GBDT         0.853904  0.478349  0.941200  \n",
       "MRMD_XGBoost    0.889169  0.518150  0.946159  \n",
       "MRMD_RF         0.906801  0.533913  0.940019  \n",
       "MRMD_ET         0.926952  0.581116  0.942223  \n",
       "MRMD_GBDT       0.869018  0.520463  0.938602  \n",
       "XGB_XGBoost     0.874055  0.528917  0.953794  \n",
       "XGB_RF          0.899244  0.518285  0.938090  \n",
       "XGB_ET          0.911839  0.564582  0.946946  \n",
       "XGB_GBDT        0.858942  0.485940  0.939074  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_eval_dic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "75845ce3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAosAAAFpCAYAAAD0jmIDAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAApuUlEQVR4nO3deZyVdd3/8ddHkEXQSEFTER1TxAUclVzulCa3UPmplCZqyu2SP3PNoKLbFrrNXEqy3JC04v5JM5UJkWumYpQ7SAgqiooytwuLmhAgIN/fHzMch2Eu5sxwZs4Z5vV8PObBdX2v7znzOWeGa97n+72WSCkhSZIkNWSzYhcgSZKk0mVYlCRJUibDoiRJkjIZFiVJkpTJsChJkqRMhkVJkiRlajQsRsSvImJBRMzK2B4R8YuImBsRMyNi/8KXKUmSpGLIZ2TxN8DgDWw/Bti99us84JaNL0uSJEmloNGwmFL6G/DuBrqcAPxPqvEE0CMiti9UgZIkSSqeQhyzuCMwv856dW2bJEmS2riOBXiOaKCtwXsIRsR51ExV061btwP69etXgG8vSZKkjTFt2rRFKaVeDW0rRFisBnaqs94beLOhjimlccA4gIEDB6ZnnnmmAN9ekiRJGyMiXs/aVohp6MnAmbVnRR8M/Cul9FYBnleSJElF1ujIYkRUAhVAz4ioBn4AbA6QUhoL3AscC8wFlgFntVSxkiRJal2NhsWU0qmNbE/AhQWrSJIkSSWjEMcsSlKrWrVqFdXV1axYsaLYpZSULl260Lt3bzbffPNilyJpE2JYlNTmVFdXs+WWW7LLLrsQ0dAFGdqflBKLFy+murqasrKyYpcjaRPivaEltTkrVqxgm222MSjWERFss802jrZKKjjDoqQ2yaC4Pt8TSS3BsChJzTRx4kQighdffDHX9tRTTzFo0CD22GMP+vXrx7nnnsuyZcsAuO+++xg4cCB77rkn/fr1Y+TIkcUqXZLy5jGLktq8XUbdU9Dnm3f1cXn1q6ys5NBDD6WqqorRo0fzzjvvcPLJJ1NVVcUhhxxCSok//vGPLFmyhFdffZWLLrqIe+65h379+rF69WrGjRtX0LolqSU4sihJzbB06VL+8Y9/cPvtt1NVVQXATTfdxPDhwznkkEOAmmnhk046ie22245rr72Wyy+/nLW3Oe3YsSMXXHBB0eqXpHwZFiWpGSZNmsTgwYPp27cvW2+9NdOnT2fWrFkccMABDfbf0DZJKmWGRUlqhsrKSoYNGwbAsGHDqKysLHJFktQyPGZRkppo8eLFPPzww8yaNYuI4KOPPiIiGD58ONOmTeOEE05Y7zF7770306ZNY9999y1CxZLUfI4sSlIT3XnnnZx55pm8/vrrzJs3j/nz51NWVsaRRx7J+PHjefLJJ3N977jjDt5++22++c1v8uMf/5iXXnoJgDVr1jBmzJhivQQp00MPfzr3JYEji1KLqbujPeLwV4pYiQqtsrKSUaNGrdP2pS99iaqqKqqqqhg5ciQLFixgs802Y9CgQXzxi1/kU5/6FNdffz2nnnoqy5YtIyI47rj8zrqWpGIyLEpq8/K91E2hTJkyZb22Sy65JLc8derUBh83ZMgQhgwZ0lJlSSqC9jAw0K7DYnv4AReS75dUGj744Lnc8lZb9S9iJZLag3YdFiVpQwxlmwY/6Eobx7BYgtr7ju2m8x/OLV849vBG+7f396upSvX9KtW6lD9/hk3j+9U0vl9NU8j3y7OhJUmSlMmwKEmSpEyGRUmSJGUyLEpSM3To0IHy8vLc17x581i8eDGf//zn6d69OxdddFGxS2x3qkdNzX1JKhxPcJHU9o3+RIGf71+NdunatSszZsxYp+3f//43V1xxBbNmzWLWrFmFrUmSisSRRUkqkG7dunHooYfSpUuXYpciSQXjyKIkNcPy5cspLy8HoKysjIkTJxa3oDZul1H35JZb+448kjbMsChJzdDQNLQkbYoMi0VU9yDs3lcfVsRKiu+Ffnt+vFJxU/EKkSSpAe35b7ZhsZmcMpGapz3vcDcV/gzVlvj3euNtUmGxqbeJa4/qjuA9XGcEz/ercf5+KR+77LILH3zwAStXrmTSpEn85S9/Ya+99ip2WWoFpbqPKNW61HZsUmGxvSvUpydHDdSQkv50nselbgpt6dKlDbbPmzevdQuRpBa2SYTF3GiZx7pJ0gaVdOiXcMCiFJV8WHTHJkmSVDwlHxYLwU8pkiS1LAd3Nl3tIixKzeUlfVRMK6uX5JY79d6yiJWorXHfpUIyLKrF+WlT7dHM6vdzywN69yhaHWpZhjK1B4ZFtRkeTtA0vl+SpEIwLEpSM3To0IH+/fvn1idNmsTLL7/MqFGjWLlyJR9FBy67/L856LODililJG08w2ILqTs1seeLLxSxEmUp1etSOm3fdP3H92+8UxM8N/y5Rvs0dG/o9957jz//+c/ssMMO/PGvj/G1r5zEX595vqC1Se4jCs+/2RtmWJSkAtlvv/1yy7vtsScrP1zByg8/LGJFUjYPVWlbinknHsOiJDXD8uXLKS8vB6CsrIyJEyeus/2v906m3z4D6NS5cxGqUyE4gifVMCxKUjM0NA291uzZs7n+x6MZO+Gu1i1qE9aWpwkdwVNzlcrZ9oZFSSqg6upqhg4dyo+uv4WddinbYN/ls2bllrvus09LlyZJzWJYlKQCef/99znuuOO46qqr2P0zBxe7HEkqCMOipIJpy1OFhXDjjTcyd+5crrjiClasGg3ALRPugjZ2Ue61P8f2+DOUWkupXpGjIYZFSW3azOr3mXBEzc4y604pdad7l2zZJ7e87c5bNfv7Ll26dL227373u3z3u9/N1SVpXe39A2VbZViUJLwPsyRl2azYBUiSJKl0ObIotUFO5UiSWothsRUU86rrktqGBa9/kFvemGMpJW0c/2avz2loSZIkZTIsSpIkKZNhUZKa4Z133uG0005j11135YADDuCQQw5h4sSJTJkyhYjgrqr/l+v77LPPEhH89Kc/LWLFktQ8HrMoqc3b/MhDAGjqqT6LM9obO2kopcSJJ57I8OHD+e1vfwvA66+/zuTJk/nkJz9J//79eWDyRL447AwAqqqq2HfffZtYnSSVBkcWJamJHn74YTp16sT555+fa9t55525+OKLAejTpw8rP1zB4oULSClx//33c8wxx+T6vvLKKwwePJj/+PKXOXL4cOa8+ioAD/z1PgafcDj77bcfRx55JO8sXADAFWN+zNlnn01FRQW77rorY8dOaMVXK6m9MyxKUhPNnj2b/ffff4N9jjzuBP5yzyRmPPMk+++/P507d85tO++887jhhht47Pe/58cjRvD1K68E4KDPHMx9kx7i2WefZdiwYVx3y/W5x7z44os88MADPPXUU1x99VhWrVrVIq9NkupzGlqSNtKFF17I3//+dzp16sRPfvITAL4w5ES+dcHZvDb3Zc4+7VQee+wxoOY2gY899hgnn3wya1asAGDlypUAvPnWm5x30Vksfm8hK1euZOftd8p9j+OOO47OnTvTuXNnevXamgULFrPjjp9q5VcqqT1yZFGSmmjvvfdm+vTpufWbbrqJhx56iIULF+baem67HR0335wnpk7hiCOOyLWvWbOGHj16MGPGDJ68806evPNOnp08GYDLf/Atzj7zPJ577jluvfVWPvzww9zj6o5MduiwGatXf9SCr1CSPmZYlKQmOvzww1mxYgW33HJLrm3ZsmXr9btgxHf4+ndG06FDh1zbVlttRVlZGX/4wx+AmpNlZs6ZA8AHS/7F9p/aHoDx48e35EuQpLzlNQ0dEYOBnwMdgNtSSlfX2/4J4A6gT+1z/jSl9OsC1yqpAaV6t4FSrasQIoJJkyZx2WWXce2119KrVy+6devGNddcs06/8oEHNfj4CRMm8LWvfY0rXnuNVatXc9LgwVw88ChGfv07nHvBcPpcsxMHH3wwrzK3NV5OplL9GZZqXWoaf45tR6NhMSI6ADcBRwHVwNMRMTml9HydbhcCz6eU/k9E9ALmRMSElNLKFqlakupY9dfHARjQu0eD25fPmpVbXrJln9xy3dvqraxe0qTvuf3221NVVdXgtoqKCmZWv79O2+jRo3PLZWVl3H///evWBRxz9HEcc/RxubrW1vS9b/wXnXpvmev7xBMTm1SrJG2MfKahDwTmppRerQ1/VcAJ9fokYMuICKA78C6wuqCVSpIkqdXlExZ3BObXWa+ubavrRmBP4E3gOeDSlNKaglQoSZKkosknLEYDbane+heAGcAOQDlwY0RsVa8PEXFeRDwTEc/UPWtQkiRJpSmfsFgN7FRnvTc1I4h1nQXclWrMBV4D+tV/opTSuJTSwJTSwF69ejW3ZkmSJLWSfMLi08DuEVEWEZ2AYcDken3eAI4AiIjtgD2AVwtZqCRJklpfo2dDp5RWR8RFwAPUXDrnVyml2RFxfu32scAVwG8i4jlqpq2/nVJa1IJ1S5IkqRXkdVHulNK9KaW+KaVPp5SurG0bWxsUSSm9mVI6OqXUP6W0T0rpjpYsWpKK7corr2TvvfdmwIABlJeX8+STT1JRUcEee+xBeXk5X/7CYYz4v8OBmsvmbLHFFixYsCD3+O7duxerdElqEu8NLanNm/qjmlvvTc2r94JGezR2geDHH3+cu+++m+nTp9O5c2cWLVqUu7/zhAkTGDhw4HrXWezZsyfXXXfdehfulqRS5+3+JKmJ3nrrLXr27Jm7X3PPnj3ZYYcdNviYs88+m9/97ne8++67rVGiJBWMYVGSmujoo49m/vz59O3blwsuuIBHH300t+3000/PTUOP+dH3cu3du3fn7LPP5uc//3kxSpakZnMaWpKaqHv37kybNo2pU6fyyCOPcMopp3D11VcD2dPQAJdccgnl5eWMGDGilSuWpOYzLEpSM3To0IGKigoqKiro378/48ePb/QxPXr04LTTTuPmm29uhQolqTAMi5LURHPmzGGzzTZj9913B2DGjBnsvPPOzJo1q9HHfuMb3+Azn/kMq1evbukyJakgPGZRkppo6dKlDB8+nL322osBAwbw/PPPM3r0aGDdYxbPO/XE9R7bs2dPhg4dyocffti6RUtSMzmyKKnNO+y7+wMwoHePBrcvrzPit2TLPrnlbXf++Bb2K6uX5P39DjjgAB577LH12qdMmZJbrnvM4togudaYMWMYM2bMOnVJUqlyZFGSJEmZDIuSJEnKZFiUJElSJsOiJEmSMhkWJUmSlMmwKEmSpEyGRUlqhg4dOlBeXs4+++zDySefzLJly9Zp/+IRh3DxWcP44F//AmDevHl07dqV8vLy3NeEyZMBWLpsGSO/cykHDtqXQUcdxKBBg3jyySeL9tokqS6vs9gO9B/fP7f8+yLWIbWUB0d8pebfAj3fiN/d3Wifrl27MmPGDKDmQtxjx47lG9/4Rq59ZvX7fPeyr/G78b/k0Gt/BMCnP/3p3GPg4+s/XvCDH7B92Z48MeVZNttsM5Z+tIgXXngBdizQC5KkjWBYVNEYYrWpOOyww5g5c+Z67QP2/wwvvzB7g499df58nn7uOR6/6bdstlnNZM+uu+7Krrvu2qQLhUtSSzEsStJGWL16Nffddx+DBw9ep/2jjz7iqX/8jaGnfCXX9sorr1BeXp5b/+lll/HeBx8wYI896NChQ2uVLElN0mbDoqNSkopp+fLlueB32GGHcc4556zT/uprr7Fn/3IOHvT53GMamoa++5FHWrNsSWoyT3CRpGZYe2zijBkzuOGGG+jUqdM67fc9PpNVK1dSNf62DT7PXrvtxnNz5rBmzZrWKFuSmsywKEktYMutPsG3//sa/ufWG1i1alVmv1132on9996ba3/2Y1JKALz88sv86U9/aq1SJWmDDIuS1EL23GcAfffah6qqKuDjYxbXft08YQIAN//whyxY+A4Hfa6cz33hEL761a+yww47FLN0Scpps8csStJaR113BwADevdocPvaS9QALNmyT2552523yi039czjpUuX5tV+w6+rcnUtX768wbq26t6dMVffsF5dng0tqRQ4sihJkqRMjiwWkGdoS1LhuE9VS/L3K3+OLEqSJCmTYVGSJEmZnIaW6nFqQpKkjxkWJUnaBPhBVy3FaWhJaoYrr7ySvffemwEDBlBeXs6TTz5JRUUFe+yxB/vuuy/Dh36Bea+8DJBrX3t9xZNOOin3PBMmT2bg0KEMOuogDjvyQG4e94tivSRJapAji5LavK1vfA6A6rx6v5dbyurf++rDNvgMjz/+OHfffTfTp0+nc+fOLFq0iJUrVwIwYcIEBg4cyPevuZ4xP/o+v/h15TrtdT0wdSo33nEHk2+9lS0/PZAVK1bwh4lVeb0KFY8jeGpvHFmUpCZ666236NmzJ507dwagZ8+e691x5YCD/oP5r7+2wef56W23cdWIEeyw7bYAdOnShTNO/c8WqVmSmsuwKElNdPTRRzN//nz69u3LBRdcwKOPPrpen0f/ej+77bFXbv3000/PTUN/85vfBOD5uXPZb6+91nusJJUSp6ElbZT2OCXXvXt3pk2bxtSpU3nkkUc45ZRTuPrqq4GaUNi1a1e23m5HRl1xTe4xDU1DS1JbYFiUpGbo0KEDFRUVVFRU0L9/f8aPHw98HApnVr/f6HPsudtuPPv881QcdFALV9u49hj6JeXHaWhJaqI5c+bw8ssv59ZnzJjBzjvvnNdjZy+anfsaec45XD5mDG8vWgTAhx9+yC9/PbZFapak5nJkUZKaaOnSpVx88cW8//77dOzYkd12241x48atc0mc+tZOT69YvYJPbvNJbvvjbQweNIgFixcz5Ktf5aPoCBGc9uWvtOIrkaTGGRYltXnvXlQzhTqgd48Gty+fNSu3vGTLPrnlbXfeKre8snpJ3t/vgAMO4LHHHluvfcqUKQ32r9s+e9HsdbadOXQoZw4duk5dklRKDIuSJKnFeDxs2+cxi5IkScpkWJQkSVImp6ElSZJKRClO2zuyKEmSpEyOLEraJNU963jXItYhSW2dI4tSG9F/fP/cl4ovIjjjjDNy66tXr6ZXr14MGTIk1/b3Rx7k1GM/z5577km/fv0YOXJkbtuffvcnBg4dygEnnsj+J5zAzeN+0ar1S1K+HFmU1Obdddv1Nf8W6PlGjx7daJ9u3boxa9Ysli9fTteuXXnwwQfZcccdc9tffvF5rvrut7hx/O84oeJAVq9ezbhx4wCY+tep3HHrHfz51lvZYdttWfHhh/zqwX8UqHpJKixHFiWpmY455hjuueceACorKzn11FNz234z9hece/EIynbrC0DHjh254IILALjt57cx8ocj2WHbbQHo0rkzZ5z6n61bvCTlybAoSc00bNgwqqqqWLFiBTNnzuSggw7KbZs75wX2GrBvg497+cWX2WvfvVqrTEnaKIZFSWqmAQMGMG/ePCorKzn22GOLXY4ktQjDoiRthOOPP56RI0euMwUN8Om+/Xh+5j8bfMxue+zG8/98vjXKk6SNZliUpI1w9tln8/3vf5/+/dc9S/0/z7+Y228cw7xX5wKwZs0axowZA8C5l57LdT+8jrcXLQLgw5Ur+eWvx7Zu4ZKUJ8+GlqSN0Lt3by699NL12vvuuQ/fHP1jRl10Lt9e9SERwXHHHQfAoKMGsXjhYoZ89auklIgIThl2VmuXLkl5MSxKavO+eO7XARjQu0eubZ2Lcr+dcstLtuyTW952561yyyurlzTpey5dunS9toqKCioqKnLrnztyMJ87cnCDdQ09bSgjDj+xwbokqZQ4DS1JkqRMhkVJkiRlMixKkiQpU15hMSIGR8SciJgbEaMy+lRExIyImB0Rjxa2TEmSJBVDoye4REQH4CbgKKAaeDoiJqeUnq/TpwdwMzA4pfRGRGzbQvVKkiSpFeUzsnggMDel9GpKaSVQBZxQr89pwF0ppTcAUkoLClumJEmSiiGfsLgjML/OenVtW119gU9GxJSImBYRZxaqQEkqVRMnTiQiePHFFwGYMmUKQ4YMWafP9y67gDvvvBOAVatW8bP//hnHHngsJx52IoedeioPTJ3a6nVLUlPkc53FaKAt1VvvCBwAHAF0BR6PiCdSSi+t80QR5wHnAfTp4zXFJBXGwpcOAOChlxre/nbdlbrzHq803P+I3hkb6qmsrOTQQw+lqqqK0aNHN9r/e9/7HgvfWcikqZPo1LkT3WYt5O/PPJPX95KkYslnZLEa2KnOem/gzQb63J9S+ndKaRHwN2Df+k+UUhqXUhqYUhrYq1ev5tYsSUW3dOlS/vGPf3D77bdTVVXVaP9ly5bxy1/+kv+6+r/o1LkTANv17MmXBg9u6VLbntGfqPmSVBLyCYtPA7tHRFlEdAKGAZPr9fkTcFhEdIyILYCDgBcKW6oklY5JkyYxePBg+vbty9Zbb8306dM32H/u3Ln06dOH7lt2b6UKJakwGg2LKaXVwEXAA9QEwN+nlGZHxPkRcX5tnxeA+4GZwFPAbSmlWS1XtiQVV2VlJcOGDQNg2LBhVFZWEtHQUTtktktSW5DXvaFTSvcC99ZrG1tv/SfATwpXmiSVpsWLF/Pwww8za9YsIoKPPvqIiODMM8/kvffeW6fvv95/j549e7Lbbrvxxhtv8O+l/6Zb925FqlySms47uEhSE915552ceeaZvP7668ybN4/58+dTVlbGu+++y5tvvskLL9QchfNm9Ru89MIsysvL2WKLLTjnnHO46jtXsWrlKgDeWriQyj//uZgvRZIaldfIotqvumd4HjaoeHVIpaSyspJRo9a9mdWXvvQlqqqquOOOOzjrrLN4f8m/6dhxc35w7S/4xCdqTtb40Y9+xPkjzuf4zx5P5y6d6bF5V7534YXFeAmSlDfD4qaq7pmEZV6mSJu2Xn2nATCgd49c2+xFs3PLu7798dW+lmz58f+HbXfeKre8snpJ3t9vypQp67VdcsklueUnnniCmdXvr9enU6dOjPjBCEb8YMR6damw/KArFY7T0JIkScrkyGIhrB3FcwRPkqTS5axbsziyKEmSpEyGRUltUkoe71ef74mkltC2pqELMHxcqgc9l2pdpcr3q33r0qULixcvZptttil2KSUjpcTixYvp0qVLsUuRNnnt7W9Q2wqLkgT07t2b6upqFi5cyDvvLc+1v7Cka2757aVv55Y/+uDjx654f1VuefGyj4PV6vdW5JY7LqlpX7HinVxbly5N210Woq6GatpQXV26dKF3795NqlOSGmNYlNTmbL755pSVlQFwzKh7cu3zrj4ut/zl8V/OLf/+qtW55akVN+WWLxx7eG65etTU3HLvq/cD4KGHh+Tajjj8lSbVWIi6GqppY+uSpKYyLKp1eSaaJEltimFRUtMZ+iWp3fBsaEmSJGVyZFGS2itHiCXlwZFFSZIkZXJkUVLRtLdrlUlSW+TIoiRJkjI5sigVkCNlam/8nZc2fYZFSZJUWJ48tUkxLErgjk1Su+YIsTbEsChJLc0PI5LaMMOiJEmtpOAjeH4QUSswLEpSG+A0odojf+9Lg5fOkSRJUiZHFiVJaozTvWrHHFmUJElSJsOiJEmSMhkWJUmSlMljFqVS5nFSkqQic2RRkiRJmQyLkiRJyuQ0tNQOeGFbSVJzObIoSZKkTI4sSpIkbQJaahbJsChJklRMJX7lC8OiJNWz9tO5x3dKkmFR0qakxD+dS1JbZFiUJLVZ150yJLd8Stm3i1iJtOnybGhJkiRlMixKkiQpk2FRkiRJmQyLkiRJymRYlCRJUibDoiRJkjIZFiVJkpTJsChJkqRMhkVJkiRlMixKkiQpk2FRkiRJmbw3dCvzPqaSJLUN/s2u4ciiJEmSMhkWJUmSlMlpaAEOtUuSpIY5sihJkqRMjixKanGOXEtS2+XIoiRJkjI5sihJapSjw1L7ldfIYkQMjog5ETE3IkZtoN9nIuKjiDipcCVKkiSpWBodWYyIDsBNwFFANfB0RExOKT3fQL9rgAdaotCm8lOwJEnSxstnZPFAYG5K6dWU0kqgCjihgX4XA38EFhSwPkmSJBVRPmFxR2B+nfXq2raciNgRGAqMLVxpkiRJKrZ8wmI00JbqrV8PfDul9NEGnyjivIh4JiKeWbhwYZ4lSpIkqVjyORu6Gtipznpv4M16fQYCVREB0BM4NiJWp5Qm1e2UUhoHjAMYOHBg/cAprcdjTyVJKq58wuLTwO4RUQb8LzAMOK1uh5RS2drliPgNcHf9oChJktQQBwZKW6NhMaW0OiIuouYs5w7Ar1JKsyPi/NrtHqcoSZK0icrrotwppXuBe+u1NRgSU0r/ufFlSZIkqRR4uz9JkiRl8nZ/klRi1h6/5bFbkkqBI4uSJEnKZFiUJElSJqehpWbwMg+SpPbCsChJUjvhB101h9PQkiRJyuTIorQJcdRAklRohkVJkgrMD27alDgNLUmSpEyGRUmSJGUyLEqSJCmTYVGSJEmZDIuSJEnKZFiUJElSJsOiJEmSMnmdRUmSpDakta/j6ciiJEmSMhkWJUmSlMlpaEntlrdkk6TGObIoSZKkTIZFSZIkZTIsSpIkKZNhUZIkSZkMi5IkScpkWJQkSVImw6IkSZIyGRYlSZKUybAoSZKkTIZFSZIkZTIsSpIkKZNhUZIkSZkMi5IkScpkWJQkSVImw6IkSZIyGRYlSZKUybAoSZKkTIZFSZIkZTIsSpIkKZNhUZIkSZkMi5IkScpkWJQkSVImw6IkSZIyGRYlSZKUybAoSZKkTIZFSZIkZTIsSpIkKZNhUZIkSZkMi5IkScpkWJQkSVImw6IkSZIyGRYlSZKUybAoSZKkTIZFSZIkZTIsSpIkKZNhUZIkSZnyCosRMTgi5kTE3IgY1cD20yNiZu3XYxGxb+FLlSRJUmtrNCxGRAfgJuAYYC/g1IjYq16314DPpZQGAFcA4wpdqCRJklpfPiOLBwJzU0qvppRWAlXACXU7pJQeSym9V7v6BNC7sGVKkiSpGPIJizsC8+usV9e2ZTkHuG9jipIkSVJp6JhHn2igLTXYMeLz1ITFQzO2nwecB9CnT588S5QkSVKx5DOyWA3sVGe9N/Bm/U4RMQC4DTghpbS4oSdKKY1LKQ1MKQ3s1atXc+qVJElSK8onLD4N7B4RZRHRCRgGTK7bISL6AHcBZ6SUXip8mZIkSSqGRqehU0qrI+Ii4AGgA/CrlNLsiDi/dvtY4PvANsDNEQGwOqU0sOXKliRJUmvI55hFUkr3AvfWaxtbZ/lc4NzCliZJkqRi8w4ukiRJymRYlCRJUibDoiRJkjIZFiVJkpTJsChJkqRMhkVJkiRlMixKkiQpk2FRkiRJmQyLkiRJymRYlCRJUibDoiRJkjIZFiVJkpTJsChJkqRMhkVJkiRlMixKkiQpk2FRkiRJmQyLkiRJymRYlCRJUibDoiRJkjIZFiVJkpTJsChJkqRMhkVJkiRlMixKkiQpk2FRkiRJmQyLkiRJymRYlCRJUibDoiRJkjIZFiVJkpTJsChJkqRMhkVJkiRlMixKkiQpk2FRkiRJmQyLkiRJymRYlCRJUibDoiRJkjIZFiVJkpTJsChJkqRMhkVJkiRlMixKkiQpk2FRkiRJmQyLkiRJymRYlCRJUibDoiRJkjIZFiVJkpTJsChJkqRMhkVJkiRlMixKkiQpk2FRkiRJmQyLkiRJymRYlCRJUibDoiRJkjIZFiVJkpTJsChJkqRMhkVJkiRlMixKkiQpk2FRkiRJmfIKixExOCLmRMTciBjVwPaIiF/Ubp8ZEfsXvlRJkiS1tkbDYkR0AG4CjgH2Ak6NiL3qdTsG2L326zzglgLXKUmSpCLIZ2TxQGBuSunVlNJKoAo4oV6fE4D/STWeAHpExPYFrlWSJEmtLJ+wuCMwv856dW1bU/tIkiSpjYmU0oY7RJwMfCGldG7t+hnAgSmli+v0uQe4KqX099r1h4BvpZSm1Xuu86iZpgbYA5hTqBdST09gUQs998awrqYpxbpKsSawrqayrqaxrqaxrvyVYk3QPuvaOaXUq6ENHfN4cDWwU5313sCbzehDSmkcMC6P77lRIuKZlNLAlv4+TWVdTVOKdZViTWBdTWVdTWNdTWNd+SvFmsC66stnGvppYPeIKIuITsAwYHK9PpOBM2vPij4Y+FdK6a0C1ypJkqRW1ujIYkppdURcBDwAdAB+lVKaHRHn124fC9wLHAvMBZYBZ7VcyZIkSWot+UxDk1K6l5pAWLdtbJ3lBFxY2NI2SotPdTeTdTVNKdZVijWBdTWVdTWNdTWNdeWvFGsC61pHoye4SJIkqf3ydn+SJEnKtEmFxcZuS1gsEfGriFgQEbOKXctaEbFTRDwSES9ExOyIuLTYNQFERJeIeCoi/llb1w+LXVNdEdEhIp6NiLuLXctaETEvIp6LiBkR8Uyx61krInpExJ0R8WLt79khJVDTHrXv09qvDyLi6yVQ12W1v++zIqIyIroUuyaAiLi0tqbZxX6fGtqPRsTWEfFgRLxc++8nS6Cmk2vfrzURUZSzaTPq+knt/8WZETExInqUSF1X1NY0IyL+EhE7lEJddbaNjIgUET1Loa6IGB0R/1tnH3Zsa9SyyYTFPG9LWCy/AQYXu4h6VgMjUkp7AgcDF5bI+/UhcHhKaV+gHBhce4Z9qbgUeKHYRTTg8yml8hK71MPPgftTSv2AfSmB9y2lNKf2fSoHDqDmhLyJxawpInYELgEGppT2oeZEwmHFrAkgIvYBvkrNXbz2BYZExO5FLOk3rL8fHQU8lFLaHXiodr3YNc0Cvgj8rZVrqes3rF/Xg8A+KaUBwEvAd1q7KBqu6ycppQG1/yfvBr7f2kWR8Tc6InYCjgLeaO2Cav2GhrPDz9bux2rPKWlxm0xYJL/bEhZFSulvwLvFrqOulNJbKaXptctLqPlDXvS77tTeMnJp7ermtV8lcWBtRPQGjgNuK3YtpS4itgIGAbcDpJRWppTeL2pR6zsCeCWl9HqxC6HmZMOuEdER2IIGrlNbBHsCT6SUlqWUVgOPAkOLVUzGfvQEYHzt8njgxGLXlFJ6IaXUUjecyEtGXX+p/TkCPEHN9ZBLoa4P6qx2owj7+w38jf4Z8C2K9DeolLLDphQWveVgM0XELsB+wJNFLgXITfXOABYAD6aUSqIu4HpqdhxrilxHfQn4S0RMi5q7JJWCXYGFwK9rp+1vi4huxS6qnmFAZbGLSCn9L/BTakYv3qLmOrV/KW5VQM0I2aCI2CYitqDm8mg7NfKY1rbd2mv61v67bZHraSvOBu4rdhFrRcSVETEfOJ3ijCyuJyKOB/43pfTPYtfSgItqp+5/1VqHXmxKYTEaaCuJEalSFhHdgT8CX6/3Ca9oUkof1U5J9AYOrJ0OK6qIGAIsqH8LyxLx2ZTS/tQcgnFhRAwqdkHUjJTtD9ySUtoP+DetP0WYKWpuMHA88IcSqOWT1IyQlQE7AN0i4ivFrapmhAy4hprpy/uBf1Jz+IrasIi4nJqf44Ri17JWSunylNJO1NR0UbHrqf1wdDklElzruQX4NDWHab0FXNca33RTCot53XJQH4uIzakJihNSSncVu576aqctp1Aax3t+Fjg+IuZRc4jD4RFxR3FLqpFSerP23wXUHH93YHErAmr+P1bXGRW+k5rwWCqOAaanlN4pdiHAkcBrKaWFKaVVwF3AfxS5JgBSSrenlPZPKQ2iZjrs5WLXVM87EbE9QO2/C4pcT0mLiOHAEOD0VJrXzfst8KViF0FNGCsD/lm7z+8NTI+ITxW1KiCl9E7tgMoa4Je00v5+UwqL+dyWULUiIqg5nuyFlNKYYtezVkT0WnuWXkR0peYP6YtFLQpIKX0npdQ7pbQLNb9bD6eUij76ExHdImLLtcvA0dRMHxZVSultYH5E7FHbdATwfBFLqu9USmAKutYbwMERsUXt/8sjKIGTgQAiYtvaf/tQc9JGqbxna00GhtcuDwf+VMRaSlpEDAa+DRyfUlpW7HrWqnfS1PGUxv7+uZTStimlXWr3+dXA/rX7taJa++Go1lBaaX+f1x1c2oKs2xIWuSwAIqISqAB6RkQ18IOU0u3FrYrPAmcAz9UeHwjwX611ZtUGbA+Mrz27fTPg9ymlkrlMTQnaDphYkzHoCPw2pXR/cUvKuRiYUPvh7VVK5DagtVNMRwH/t9i1AKSUnoyIO4Hp1EwPPkvp3D3ijxGxDbAKuDCl9F6xCmloPwpcDfw+Is6hJnSfXAI1vQvcAPQC7omIGSmlL5RAXd8BOgMP1u4vnkgpnV8CdR1b+6FyDfA60Ko1ZdVVAn+js96viogop+Ywu3m00n7MO7hIkiQp06Y0DS1JkqQCMyxKkiQpk2FRkiRJmQyLkiRJymRYlCRJUibDoiRJkjIZFiVJkpTJsChJkqRM/x/jb3e6q2iJlQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 792x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "all_eval.plot.bar(figsize=(11,6),rot=0,width = 0.7,grid = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86ce9081",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}