All_Classifiers.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import sklearn\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import preprocessing\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.model_selection import cross_validate\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn import svm , preprocessing\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.experimental import enable_hist_gradient_boosting\n",
    "from sklearn.ensemble import HistGradientBoostingClassifier\n",
    "import pickle\n",
    "from lightgbm import LGBMClassifier\n",
    "from catboost import CatBoostClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mean_score(scoring):\n",
    "    return {i:j.mean() for i,j in scoring.items()}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading data.\n",
    "- Load data and shuffle it with specific seeding."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>having_IPhaving_IP_Address</th>\n",
       "      <th>URLURL_Length</th>\n",
       "      <th>Shortining_Service</th>\n",
       "      <th>having_At_Symbol</th>\n",
       "      <th>double_slash_redirecting</th>\n",
       "      <th>Prefix_Suffix</th>\n",
       "      <th>having_Sub_Domain</th>\n",
       "      <th>SSLfinal_State</th>\n",
       "      <th>Domain_registeration_length</th>\n",
       "      <th>Favicon</th>\n",
       "      <th>...</th>\n",
       "      <th>popUpWidnow</th>\n",
       "      <th>Iframe</th>\n",
       "      <th>age_of_domain</th>\n",
       "      <th>DNSRecord</th>\n",
       "      <th>web_traffic</th>\n",
       "      <th>Page_Rank</th>\n",
       "      <th>Google_Index</th>\n",
       "      <th>Links_pointing_to_page</th>\n",
       "      <th>Statistical_report</th>\n",
       "      <th>Result</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>8155</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5323</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8926</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4737</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10348</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       having_IPhaving_IP_Address  URLURL_Length  Shortining_Service  \\\n",
       "index                                                                  \n",
       "8155                            1             -1                   1   \n",
       "5323                            1              1                   1   \n",
       "8926                            1             -1                   1   \n",
       "4737                            1             -1                   1   \n",
       "10348                          -1             -1                   1   \n",
       "\n",
       "       having_At_Symbol  double_slash_redirecting  Prefix_Suffix  \\\n",
       "index                                                              \n",
       "8155                  1                         1             -1   \n",
       "5323                 -1                         1              1   \n",
       "8926                  1                         1              1   \n",
       "4737                  1                         1             -1   \n",
       "10348                 1                         1             -1   \n",
       "\n",
       "       having_Sub_Domain  SSLfinal_State  Domain_registeration_length  \\\n",
       "index                                                                   \n",
       "8155                   1               0                           -1   \n",
       "5323                   0               1                           -1   \n",
       "8926                   0               1                           -1   \n",
       "4737                   1              -1                            1   \n",
       "10348                  0               1                           -1   \n",
       "\n",
       "       Favicon  ...  popUpWidnow  Iframe  age_of_domain  DNSRecord  \\\n",
       "index           ...                                                  \n",
       "8155         1  ...            1       1             -1          1   \n",
       "5323         1  ...            1       1              1          1   \n",
       "8926         1  ...            1       1             -1          1   \n",
       "4737         1  ...            1       1             -1          1   \n",
       "10348        1  ...            1       1              1          1   \n",
       "\n",
       "       web_traffic  Page_Rank  Google_Index  Links_pointing_to_page  \\\n",
       "index                                                                 \n",
       "8155            -1         -1             1                       0   \n",
       "5323             0          1             1                       0   \n",
       "8926             0         -1             1                       0   \n",
       "4737             1         -1             1                       0   \n",
       "10348            1         -1            -1                       0   \n",
       "\n",
       "       Statistical_report  Result  \n",
       "index                              \n",
       "8155                    1      -1  \n",
       "5323                    1       1  \n",
       "8926                    1       1  \n",
       "4737                    1      -1  \n",
       "10348                   1      -1  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"dataset.csv\",index_col=0) #Reading data from CSV file\n",
    "X = df.drop(\"Result\",axis=1).values #dropping result column from df as Result is our output variable\n",
    "X = preprocessing.scale(X) #Preprocessing data to bring all values into one scale\n",
    "Y = df['Result'].values #Placing result as our output variable\n",
    "df = sklearn.utils.shuffle(df) #Shuffeling the data\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evalution metrics \n",
    " - Specifying evaluation metrics for classification models\n",
    " - Using 10 fold-cross-validation for evaluting "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "scoring = {'accuracy': 'accuracy',\n",
    "           'recall': 'recall',\n",
    "           'precision': 'precision',\n",
    "           'f1': 'f1'} #scoring array to print the result of our evaluation\n",
    "fold_count=10 #count of fold that'll be used on K-fold cross validation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classication models for phishing detection:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Descision Tree\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fit_time :  0.058835959434509276\n",
      "score_time :  0.008617329597473144\n",
      "test_accuracy :  0.9618176272668164\n",
      "test_recall :  0.9675071270193222\n",
      "test_precision :  0.9641647631931616\n",
      "test_f1 :  0.9657549443903417\n"
     ]
    }
   ],
   "source": [
    "dtree_clf=DecisionTreeClassifier() #Decision Tree classifier\n",
    "cross_val_scores = cross_validate(dtree_clf, X, Y, cv=fold_count, scoring=scoring)\n",
    "dtree_score = mean_score(cross_val_scores)\n",
    "for score in dtree_score:\n",
    "    print(score,\": \", dtree_score[score])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fit_time :  0.14523100852966309\n",
      "score_time :  0.03047034740447998\n",
      "test_accuracy :  0.9716817233857686\n",
      "test_recall :  0.9803444726005701\n",
      "test_precision :  0.969261904790747\n",
      "test_f1 :  0.9747320757365662\n"
     ]
    }
   ],
   "source": [
    "rforest_clf=RandomForestClassifier() #Random Forest Classifier\n",
    "cross_val_scores = cross_validate(rforest_clf, X, Y, cv=fold_count, scoring=scoring)\n",
    "rforest_clf_score = mean_score(cross_val_scores)\n",
    "for score in rforest_clf_score:\n",
    "    print(score,\": \", rforest_clf_score[score])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ### Ada_Booster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fit_time :  0.7922648668289185\n",
      "score_time :  0.10146000385284423\n",
      "test_accuracy :  0.9325185312895744\n",
      "test_recall :  0.9533869179600887\n",
      "test_precision :  0.9275741789524385\n",
      "test_f1 :  0.9402675857045054\n"
     ]
    }
   ],
   "source": [
    "adaboost_clf=AdaBoostClassifier() #AdaBoost Classifier\n",
    "cross_val_scores = cross_validate(adaboost_clf, X, Y, cv=fold_count, scoring=scoring)\n",
    "adaboost_clf_score = mean_score(cross_val_scores)\n",
    "for score in adaboost_clf_score:\n",
    "    print(score,\": \", adaboost_clf_score[score])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gradient Boosting With XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[13:04:48] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[13:04:49] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[13:04:50] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[13:04:52] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[13:04:53] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[13:04:55] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[13:04:56] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[13:04:57] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[13:04:59] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[13:05:00] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "fit_time :  1.4121744871139525\n",
      "score_time :  0.028862524032592773\n",
      "test_accuracy :  0.9736751321967339\n",
      "test_recall :  0.9827832330271355\n",
      "test_precision :  0.9704323537241274\n",
      "test_f1 :  0.976544997591574\n"
     ]
    }
   ],
   "source": [
    "XGB_clf=XGBClassifier() #XGboost Classifier\n",
    "cross_val_scores = cross_validate(XGB_clf, X, Y, cv=fold_count, scoring=scoring)\n",
    "XGB_clf_score= mean_score(cross_val_scores)\n",
    "for score in XGB_clf_score:\n",
    "    print(score,\": \", XGB_clf_score[score])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gradient_Booster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fit_time :  2.1573262453079223\n",
      "score_time :  0.022116541862487793\n",
      "test_accuracy :  0.9470810612251335\n",
      "test_recall :  0.9615082884595079\n",
      "test_precision :  0.9445959161957547\n",
      "test_f1 :  0.9529350226081249\n"
     ]
    }
   ],
   "source": [
    "gradientBooster_clf=GradientBoostingClassifier() #Gradient Boosting Classifier\n",
    "cross_val_scores = cross_validate(gradientBooster_clf,X, Y, cv=fold_count, scoring=scoring)\n",
    "gradientBooster_clf_score= mean_score(cross_val_scores)\n",
    "for score in gradientBooster_clf_score:\n",
    "    print(score,\": \", gradientBooster_clf_score[score])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Histogram-Based Gradient Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fit_time :  2.8998016357421874\n",
      "score_time :  0.05566966533660889\n",
      "test_accuracy :  0.9690618735322756\n",
      "test_recall :  0.9775886917960088\n",
      "test_precision :  0.9674153303406949\n",
      "test_f1 :  0.9724148355232094\n"
     ]
    }
   ],
   "source": [
    "histGradientBooster_clf = HistGradientBoostingClassifier() #Histogram Based Gradient Boosting Classifier\n",
    "cross_val_scores = cross_validate(histGradientBooster_clf,X, Y, cv=fold_count, scoring=scoring)\n",
    "histGradientBooster_clf_score= mean_score(cross_val_scores)\n",
    "for score in histGradientBooster_clf_score:\n",
    "    print(score,\": \", histGradientBooster_clf_score[score])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LightGBM for Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fit_time :  0.4222498893737793\n",
      "score_time :  0.039673876762390134\n",
      "test_accuracy :  0.9681560761343613\n",
      "test_recall :  0.9749889135254989\n",
      "test_precision :  0.9682286477486434\n",
      "test_f1 :  0.9715499984612388\n"
     ]
    }
   ],
   "source": [
    "LGBM_clf = LGBMClassifier() #Light Gradient Boosting Classifier\n",
    "cross_val_scores = cross_validate(LGBM_clf,X, Y, cv=fold_count, scoring=scoring)\n",
    "LGBM_clf_score= mean_score(cross_val_scores)\n",
    "for score in LGBM_clf_score:\n",
    "    print(score,\": \", LGBM_clf_score[score])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ### Gradient Boosting with CatBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fit_time :  1.3329977512359619\n",
      "score_time :  0.08873751163482665\n",
      "test_accuracy :  0.9716844262576967\n",
      "test_recall :  0.9835951853025025\n",
      "test_precision :  0.9662799514037845\n",
      "test_f1 :  0.9748327345474397\n"
     ]
    }
   ],
   "source": [
    "catBoost_clf = CatBoostClassifier(verbose=0, n_estimators=100) #CatBoost Classifier\n",
    "cross_val_scores = cross_validate(catBoost_clf,X, Y, cv=fold_count, scoring=scoring)\n",
    "catBoost_clf_score= mean_score(cross_val_scores)\n",
    "for score in catBoost_clf_score:\n",
    "    print(score,\": \", catBoost_clf_score[score])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fit_time :  0.23046772480010985\n",
      "score_time :  3.310789179801941\n",
      "test_accuracy :  0.9534988715650521\n",
      "test_recall :  0.9629635202196178\n",
      "test_precision :  0.954057726671312\n",
      "test_f1 :  0.9584625042863312\n"
     ]
    }
   ],
   "source": [
    "KNeighbors_clf=KNeighborsClassifier(3) #K-Nearest Neighbours Classifiers. K=3\n",
    "cross_val_scores = cross_validate(KNeighbors_clf, X, Y, cv=fold_count, scoring=scoring)\n",
    "KNeighbors_clf_score = mean_score(cross_val_scores)\n",
    "for score in KNeighbors_clf_score:\n",
    "    print(score,\": \", KNeighbors_clf_score[score])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regrression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fit_time :  0.19339196681976317\n",
      "score_time :  0.008561182022094726\n",
      "test_accuracy :  0.9255528222465482\n",
      "test_recall :  0.9438045084996304\n",
      "test_precision :  0.9242416485785391\n",
      "test_f1 :  0.9338912261628091\n"
     ]
    }
   ],
   "source": [
    "logistic_clf=LogisticRegression(random_state=1) #Logistic Regression Classifier\n",
    "cross_val_scores = cross_validate(logistic_clf, X, Y, cv=fold_count, scoring=scoring)\n",
    "logistic_clf_score = mean_score(cross_val_scores)\n",
    "for score in logistic_clf_score:\n",
    "    print(score,\": \", logistic_clf_score[score])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fit_time :  14.381748700141907\n",
      "score_time :  0.008480691909790039\n",
      "test_accuracy :  0.9704168014100352\n",
      "test_recall :  0.9813200823566677\n",
      "test_precision :  0.9661992306515164\n",
      "test_f1 :  0.9736681327772573\n"
     ]
    }
   ],
   "source": [
    "neural_clf=MLPClassifier(hidden_layer_sizes=(33,),max_iter=500) #Multi-layer Perceptron Neural Network\n",
    "cross_val_scores = cross_validate(neural_clf, X, Y, cv=fold_count, scoring=scoring)\n",
    "neural_clf_score = mean_score(cross_val_scores)\n",
    "for score in neural_clf_score:\n",
    "    print(score,\": \", neural_clf_score[score])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For Liner Kernel\n",
      "fit_time :  2.902037763595581\n",
      "score_time :  0.308991527557373\n",
      "test_accuracy :  0.9270906294866281\n",
      "test_recall :  0.9468889240840459\n",
      "test_precision :  0.9241913584252026\n",
      "test_f1 :  0.9353628640634943\n",
      "For Polynomial Kernel\n",
      "fit_time :  2.0356573343276976\n",
      "score_time :  0.5117424488067627\n",
      "test_accuracy :  0.9491615287789896\n",
      "test_recall :  0.9704392355611869\n",
      "test_precision :  0.9402992293771444\n",
      "test_f1 :  0.955101981004332\n",
      "For rbf Kernel\n",
      "fit_time :  2.010216975212097\n",
      "score_time :  0.5641074657440186\n",
      "test_accuracy :  0.9509711572703253\n",
      "test_recall :  0.9693028719248232\n",
      "test_precision :  0.9442490472181699\n",
      "test_f1 :  0.9565849587371293\n",
      "For sigmoid Kernel\n",
      "fit_time :  1.8756847620010375\n",
      "score_time :  0.589139747619629\n",
      "test_accuracy :  0.8290269632894434\n",
      "test_recall :  0.8495953436807095\n",
      "test_precision :  0.8447206092080265\n",
      "test_f1 :  0.8468919694665653\n"
     ]
    }
   ],
   "source": [
    "#linear\n",
    "print(\"For Liner Kernel\")\n",
    "linear_clf = svm.SVC(kernel='linear')\n",
    "cross_val_scores = cross_validate(linear_clf, X, Y, cv=fold_count, scoring=scoring)\n",
    "linear_svc_clf_score = mean_score(cross_val_scores)\n",
    "for score in linear_svc_clf_score:\n",
    "    print(score,\": \", linear_svc_clf_score[score])\n",
    "\n",
    "#poly\n",
    "print(\"For Polynomial Kernel\")\n",
    "poly_clf = svm.SVC(kernel='poly')\n",
    "cross_val_scores = cross_validate(poly_clf, X, Y, cv=fold_count, scoring=scoring)\n",
    "poly_svc_clf_score = mean_score(cross_val_scores)\n",
    "for score in poly_svc_clf_score:\n",
    "    print(score,\": \", poly_svc_clf_score[score])\n",
    "\n",
    "#rbf\n",
    "print(\"For rbf Kernel\")\n",
    "rbf_clf = svm.SVC(kernel='rbf')\n",
    "cross_val_scores = cross_validate(rbf_clf, X, Y, cv=fold_count, scoring=scoring)\n",
    "rbf_svc_clf_score = mean_score(cross_val_scores)\n",
    "for score in rbf_svc_clf_score:\n",
    "    print(score,\": \", rbf_svc_clf_score[score])\n",
    "\n",
    "#sigmoid\n",
    "print(\"For sigmoid Kernel\")\n",
    "sigmoid_clf = svm.SVC(kernel='sigmoid')\n",
    "cross_val_scores = cross_validate(sigmoid_clf, X, Y, cv=fold_count, scoring=scoring)\n",
    "sigmoid_svc_clf_score = mean_score(cross_val_scores)\n",
    "for score in sigmoid_svc_clf_score:\n",
    "    print(score,\": \", sigmoid_svc_clf_score[score])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}