final_project_checkin.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Final Project Check-in 2018-11-16\n",
    "------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Group Name\n",
    "-----\n",
    "Project Impossible"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Student Names\n",
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Yuhan Wang\n",
    "2. Byron Han\n",
    "3. Jinghui Zhao\n",
    "4. Liying Li\n",
    "5. Shulun Chen"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load Data\n",
    "-----"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "talking = pd.read_csv('talking.csv', parse_dates = ['click_time'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Overview\n",
    "------\n",
    "Columns:\n",
    "\n",
    "• ip: ip address of click\n",
    "\n",
    "• app: app id for marketing\n",
    "\n",
    "• device: device type id of user mobile phone (e.g., iphone 6 plus, iphone 7, huawei mate 7, etc.)\n",
    "\n",
    "• os: os version id of user mobile phone\n",
    "\n",
    "• channel: channel id of mobile ad publisher\n",
    "\n",
    "• click_time: timestamp of click (UTC)\n",
    "\n",
    "• attributed_time: if user download the app for after clicking an ad, this is the time of the app download\n",
    "\n",
    "• is_attributed: the target that is to be predicted, indicating the app was downloaded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "184903890"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of rows\n",
    "len(talking)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 184903890 entries, 0 to 184903889\n",
      "Data columns (total 8 columns):\n",
      "ip                 int64\n",
      "app                int64\n",
      "device             int64\n",
      "os                 int64\n",
      "channel            int64\n",
      "click_time         datetime64[ns]\n",
      "attributed_time    object\n",
      "is_attributed      int64\n",
      "dtypes: datetime64[ns](1), int64(6), object(1)\n",
      "memory usage: 11.0+ GB\n"
     ]
    }
   ],
   "source": [
    "talking.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ip</th>\n",
       "      <th>app</th>\n",
       "      <th>device</th>\n",
       "      <th>os</th>\n",
       "      <th>channel</th>\n",
       "      <th>click_time</th>\n",
       "      <th>attributed_time</th>\n",
       "      <th>is_attributed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>83230</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>379</td>\n",
       "      <td>2017-11-06 14:32:21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>17357</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>379</td>\n",
       "      <td>2017-11-06 14:33:34</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>35810</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>379</td>\n",
       "      <td>2017-11-06 14:34:12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>45745</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>478</td>\n",
       "      <td>2017-11-06 14:34:52</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>161007</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>379</td>\n",
       "      <td>2017-11-06 14:35:08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       ip  app  device  os  channel          click_time attributed_time  \\\n",
       "0   83230    3       1  13      379 2017-11-06 14:32:21             NaN   \n",
       "1   17357    3       1  19      379 2017-11-06 14:33:34             NaN   \n",
       "2   35810    3       1  13      379 2017-11-06 14:34:12             NaN   \n",
       "3   45745   14       1  13      478 2017-11-06 14:34:52             NaN   \n",
       "4  161007    3       1  13      379 2017-11-06 14:35:08             NaN   \n",
       "\n",
       "   is_attributed  \n",
       "0              0  \n",
       "1              0  \n",
       "2              0  \n",
       "3              0  \n",
       "4              0  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "talking.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ip</th>\n",
       "      <th>app</th>\n",
       "      <th>device</th>\n",
       "      <th>os</th>\n",
       "      <th>channel</th>\n",
       "      <th>click_time</th>\n",
       "      <th>attributed_time</th>\n",
       "      <th>is_attributed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>184903885</th>\n",
       "      <td>121312</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>340</td>\n",
       "      <td>2017-11-09 16:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>184903886</th>\n",
       "      <td>46894</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>211</td>\n",
       "      <td>2017-11-09 16:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>184903887</th>\n",
       "      <td>320126</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>274</td>\n",
       "      <td>2017-11-09 16:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>184903888</th>\n",
       "      <td>189286</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>37</td>\n",
       "      <td>259</td>\n",
       "      <td>2017-11-09 16:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>184903889</th>\n",
       "      <td>106485</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>137</td>\n",
       "      <td>2017-11-09 16:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               ip  app  device  os  channel          click_time  \\\n",
       "184903885  121312   12       1  10      340 2017-11-09 16:00:00   \n",
       "184903886   46894    3       1  19      211 2017-11-09 16:00:00   \n",
       "184903887  320126    1       1  13      274 2017-11-09 16:00:00   \n",
       "184903888  189286   12       1  37      259 2017-11-09 16:00:00   \n",
       "184903889  106485   11       1  19      137 2017-11-09 16:00:00   \n",
       "\n",
       "          attributed_time  is_attributed  \n",
       "184903885             NaN              0  \n",
       "184903886             NaN              0  \n",
       "184903887             NaN              0  \n",
       "184903888             NaN              0  \n",
       "184903889             NaN              0  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "talking.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ip</th>\n",
       "      <th>app</th>\n",
       "      <th>device</th>\n",
       "      <th>os</th>\n",
       "      <th>channel</th>\n",
       "      <th>click_time</th>\n",
       "      <th>attributed_time</th>\n",
       "      <th>is_attributed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>123635662</th>\n",
       "      <td>333473</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>340</td>\n",
       "      <td>2017-11-08 16:34:41</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>166856116</th>\n",
       "      <td>88911</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>435</td>\n",
       "      <td>2017-11-09 10:34:34</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77586606</th>\n",
       "      <td>131991</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>215</td>\n",
       "      <td>2017-11-08 02:36:08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130534557</th>\n",
       "      <td>30587</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>134</td>\n",
       "      <td>2017-11-08 23:28:44</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44656566</th>\n",
       "      <td>138871</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>237</td>\n",
       "      <td>2017-11-07 11:05:58</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               ip  app  device  os  channel          click_time  \\\n",
       "123635662  333473   12       1  19      340 2017-11-08 16:34:41   \n",
       "166856116   88911    2       1  13      435 2017-11-09 10:34:34   \n",
       "77586606   131991    9       1   8      215 2017-11-08 02:36:08   \n",
       "130534557   30587    9       1  19      134 2017-11-08 23:28:44   \n",
       "44656566   138871    2       1  19      237 2017-11-07 11:05:58   \n",
       "\n",
       "          attributed_time  is_attributed  \n",
       "123635662             NaN              0  \n",
       "166856116             NaN              0  \n",
       "77586606              NaN              0  \n",
       "130534557             NaN              0  \n",
       "44656566              NaN              0  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "talking.sample(5)  # ummm... wierd, is_attributed always 0?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imbalanced dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_attributed_time = talking['is_attributed'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    184447044\n",
       "1       456846\n",
       "Name: is_attributed, dtype: int64"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_attributed_time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "is_attributed==0 weight: 99.7529%\n",
      "is_attributed==1 weight: 0.2471%\n"
     ]
    }
   ],
   "source": [
    "# extremely imbalanced data, we may need to resample the data in the training stage\n",
    "print('is_attributed==0 weight: %.4f%%' % (num_attributed_time[0]/sum(num_attributed_time)*100))\n",
    "print('is_attributed==1 weight: %.4f%%' % (num_attributed_time[1]/sum(num_attributed_time)*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWQAAADuCAYAAAAOR30qAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3Xd8VFXex/HPmZn0QkgmQELvEGmRJgoEUIlYAosortjL6mLfotnVx3V5nt2N6+rq2l1X14YFEQULWSlGQEWQICVIUToBMpCQ3mbO88cdMLpgEsjMmZn83q9XBCaZO9/hRb6enHvvOUprjRBCCPNspgMIIYSwSCELIUSAkEIWQogAIYUshBABQgpZCCEChBSyEEIECClkIYQIEFLIQggRIKSQhRAiQEghCyFEgJBCFkKIACGFLIQQAUIKWQghAoRPClkp9fsGv09QSs1s5Os/8/46Tin1fjNfa4pSKu0kMpY39zlCCOFLvhoh/77B7xOA4xayUsoOoLU+8xReawrQ7EIWQohA4zjVAyil3gU6A5HAY0APIEoptRbYCNiBnt4/fwx8APwBKASGAGlKqXKtdaz3kPFKqXlAX+BTYKbW2tPwa5RS04ALgeeALCBDKXUfcLH3GE8CyUAlcKPW+hulVHdgtvc9LzzV9y2EEC3tlAsZuE5rfVgpFQWsAjKAW7XWQwCUUt2AAQ3+PA4Y4X1s+3GONwJrxLsTqzinAm8f74W11p8ppeYD72ut3/YefzFws9Z6q1JqJPAUMAHrfxZPa61fVkrd0gLvWwghWlRLTFncrpT6GvgCa6TcuwnP+fIEZXz0c99prd3A68DopgZRSsUCZwJzvCPyZ4EU76fP8h4P4JWmHlMIIfzllEbI3tHuOcAorXWlUuoTrKmLxlT8xOd+vKeUPs7jJ3oNG1BydDTehGMLIUTAONUpizZAsbeM+wFneB+vU0qFaa3rgDIgrhnHHOGd790JTMeaJwY4oJTqD2wGfuY9Lg2Pr7UuVUptV0pdorWeo5RSwCCt9dfACuAy4FVgxsm+YRFYcvJdiUA7IMn7kQi0xfq37QY8x/k4+ngVsA/YC+zJTndW+Tu/EA2pU9nkVCkVAbwLdMQqymTgAWAS1sm2NVrrGUqp2cAg4COsk3q/0Vpf2OA45VrrWO+I+36gCBjID0/qTQMeBHYDG4BYrfU1SqmzgH8CNcA0rG+0p7GmKsKAN7TWs350Um8ucF+DE4kigOXkuzpjnQBOxzrZ2wnr31wqENWCL1UM7MFb0N5fj/57W5ud7qxpwdcS4r+cUiEL0ZJy8l12rMI9Wr7pwGDAaTKXVx2wHuvE9dGPjdnpTrfRVCKkSCELo3LyXYOAC7B+qhpGy454fa0SyMcq5+XAx9npzlKzkUQwk0JujFJXA0vRepfpKKEgJ98VDZwNXKC1Pl8p1dl0phZUByzDmpZ7PzvducVwHhFkpJBPIEupyJcgpi1sw5qfnoLWX5jOFYxy8l1dgYuwSnicUqopV+KEggKsa+jnZKc7N5gOIwKfFPJxZCmVDPwfsO5JWN4Z5mBdY309Ws82my445OS7woDJWuubgQneK15as2+AN4DnstOdhabDiMAkhXwcWUpdBNwIHAJ2TYN/X2Xd8Xce8L/AH5C/uOPKyXd11x7PjVrrG2x2e7LpPAGoDngHeCI73bncdBgRWKSQjyNLqXDgcqxbruuAimh49A24Gfgt1oj5arSW61aBnHyXA7jIU18/U9ntE5RSsqxr06wFngBmyzXQAqSQTyjL+hF7PHAV1o0EAE/Pty7Leh7rEqjJaN1qf/zMyXclaK1v1x7PTJvd3t50niB2GHgBeCo73XmiJQVEKyCF3Igs6w7E27Eux7IDc96GwnCYByjgIrTON5nR33LyXW3qaqrvsTvCbrPZ7XJzTcvxYJ0EvDc73bnNdBjhf1LITZClVDusUu7ifWj5Q/BRX2uxonTgCrSeZyygn+Tku+Lqqqt+Z3OE3W53OGJM5wlhdVgLY83KTncWmQ4j/EcKuYmyrOVFrwVGeR/anA7P/hH+DFwP3IvWfzEW0Idy8l1hNRXldzjCI+6zh4W1MZ2nFSkDHgIeyU53/tSCXCJESCE3Q5Z1suoCrDUzbFhrbjw637rG9lGstTJuROtacylb1h+X77jC5nA8GBYRmWo6SytWiLVGzL/kVu3QJoV8ErKUGgLcAkRj3T77xHxrhbE5HF2NTuug/lHzvsXf9HKEh78eGRs/zHQWccw3wD3Z6c75poMI35BCPklZSnXCmlfuBNQDr82FzWHW6ndtsE72Bd3dWT//y7Oq04Chf0ro0PE3dkdYmOk84rjeBGZmpzsPmw4iWpYU8inIUioO+AUwFGvx+49/A++OtZYDzQQuQ+sPTWZsjl/NXXF6TGLymzEJib1MZxGN2g/cmJ3ubNYu7SKwSSGfoiylHFibq16EtdZyPtb1yrdjbeZ6N1r/3WDERt05Z5k9PCr6kYQOHW+x2R1203lEs/wbuENWmQsNUsgtwHsTySjgBqzdS7ZjnewbhrVDyevArVg7qASUX81dcVass/3s6PiELo1/tQhQu4Drs9Odi0wHEadGCrkFZSnVA7gDa165CPiH9+zLe1g7T1yM1sWm8jWUlpGpzrvjD39zdul5p81ul1udg58GngF+K5fIBS8p5BaWpVQiMBNr14tK4MU3YG20tdJXd+BCtDa6Tm7mrb9PGXjO5PecXXsON5lD+MRWYEp2urPAdBDRfFLIPpBl7TU4A+vEHsD8C2HOL6ybSK4DLkHrxSayXfLHf4zuc+Y5c+Oc7dqZeH3hF2XAVdnpzndNBxHNI4XsI9555QnANUAs1hY/z863dsx+EvgtWj/rrzxpGZlq6OTLb+19xvgHw6Oig2mbJHFyNNZSsQ9kpzvlmzxISCH7WJZS/YHbsBa434B1sq8L1uJEbwO/Rmuf3n2VlpEZPubKW/7ZdcjIK2S+uNWZB1yRne6sNB1ENE4K2Q+ylGqPVcpDsE7uPTofjmAtVF6Cdb2yTy5bGn/9ne2HTJr2fvue/eSOu9ZrNXBRdrpzv+kg4qfJaMkP5mt9AGv++CMgFbg/C3oVwjisdQo+Q6nuLf26E264a9DIadd+JWXc6g0DvsjJd51mOoj4aVLIPqaUOk8ptXkBrFsA8ViL24cDv7oJptigwAlJabDFqdRqpVTXBs91K6XWej/mN3j8NaXUOqXUnxs89j9KqclgzRePnnFT5shp136c0KFjRz++XRG4ugKf5eS7zjIdRJyYFLIPKaXsWCfwJgFpwGUL4FvgL1hTFtf0h66JMKAAJt4HvfrBWw0OUaW1HuL9yPIecxCA1noQMEYp1UYplQKM0Fq/l5aRqRI7dp069qpbX23TPlWupBANxQMf5eS7RjX6lcIIKWTfGgFs01p/p60lOd8AJs/Xeg3wP0BBT+jXD+7MgvVfwoxoGIBSOZx4l+Y6IMq7b1041vZSs4D70zIyVVLn7pdmXHvHc/HtUpx+eH8i+MQBC3PyXSNMBxH/TQrZtzpincQ7ao/3MeZrvQerSP8DDAdmzYXpm+FvQC+sE36RyprG+EIpNQVAa70J61bZNVij6V6A6j924trkbr1nZFx7x9NxzvaJfnp/IjjFA//JyXfJuYUAI4XsW8cb5R67rGW+dWXFo8BL38LIMDh/FCwCLgHWbIcCDVOwdsB+VCnVE0Brfad3GuNh4H+VzXb/gW83zznw3eYXCj5Z2Nb3b0uEgDZYpXy66SDie1LIvrUH6/rjozoB+xp+wXyt6xdA0WZIOgPyouBXWTA5C/7UzZrWWKwhCfgEa/++Y7wn8VafNv78a9r36p91zT9eD8v/4C1qq+SSU9EkbYGPc/JdQ0wHERYpZN9aBfRWSnVXSoUDlwE/2O1BKZUOPOuGc6PhbqwSv7Ec7uoPS4Gp++GlOOs27IIGzwsD7hg86eLCkdOuuSc8KjoMQGsP7vqAW1ROBK5EYFFOvmuQ6SBCCtmntNb1wK1ALrAJeEtrvVEpNUspleX9soewbq2eswDmfmDNCX9RARdvh602eLMT6HvhkLamMo66JSGl05Zzb77nDz1HjI3XWvPopWPpOngkUXGyD6loliSskXLnRr9S+JTcqReAGixO9DOstZUfmW+dHPwb0AG4Nm3sxIwJN/7m8U6npfc2GFWEljXA6Ox0Z5XpIK2VjJAD0Hyta4AXgX8AKcAfs2AwWt8FLCy1O1ac3m/Q3VLGooWdjnXjkjBERsgBLkupNOAurLm+F749a8KWCXbHP+7+ctmgp+6atSlu6pXjbHa7bLskWtLd2enOh0yHaI2kkIOAd3Gi24GB1Rde6hh73yPnlHz95Ve/vPPKdu9cNP276rv+OCosIjLGdE4RMjzA+dnpzlzTQVobKeQgMbpLj0TVNunx8X9+dkp0apdogJI9O7ZMvnlq1e6UzmUbHn6pT3R8gtwqLVpKCTAiO9251XSQ1kTmkINAWkam7XD33pcOvffh0UfLGCChU7c+C2cv6VgbHuHJvPzsfSV7d8o3j2gpCcC7OfmuONNBWhMp5OBw9tCsn1/XoXfaf+0MHRWf4Nzx91dGLJ10cdFVPz/b7fr6y9UmAoqQlAa8bDpEayKFHODSMjL7Orv0vLn/2PPST/Q1jvCISPsvs8958YHHim64dXpc8YI3PvF4PB5/5hQha0pOvusa0yFaC5lDDmBpGZnxStn+b/Lv/jo1vl1Kk9Y1Pvztpg3TZl7qXjTp4oOemb8b7QiPkP3zxKkqAdKy052FpoOEOhkhB6i0jEwFTD896+fDmlrGAIk9+w9Y8PrS5H5rV0Z3/vXVX1aXlR7yYUzROiQAz5gO0RpIIQeugYmdup/Xb8zEZi+RGJPoTN34zDuDXSmd6868ZtLW0gN7t/sioGhVsnLyXZebDhHqpJADUFpGZiyoG8ZcOXOE3eEIO5ljhEVGxdbckzN+yZUzqy6ace7hQwVf57d0TtHq/CMn39XedIhQJoUcYLxTFZeeftH0YW3ap3Y6lWPZ7HZ7wpQZ49959FU1+Y7L6w/959087fHISQNxspKwtiQTPiKFHHhOa5va5YL+Gee12G4OSQNOP/3D1xYlj37xMVvdcw8tdtfV1bTUsUWrc3FOvuuSxr9MnAwp5ACSlpEZA9ww+oqZQ+2OsJOaqjiR+HYp3dY+v2Bg0q7vaHvvTctqKsqKW/L4olV5IiffJduE+YAUcmCZ1nPEmL5tUzt39cXBI2JiE0pmPZlROGCo6n/Tz74uK9q/yxevI0JeO+A+0yFCkRRygEjLyOwNTBg8aZpPd26wOxxh0VfOPPvL2+6PHHXdhd8d3lqwzpevJ0LWLTn5ru6mQ4QaKeQAcPRE3mnjL2gX29aZ6uvXU0rhHDn2jOVPv50y/Pc3FR3KW7hcyx1ConnCgT+ZDhFqpJADwwBls/dNm3CBX7dlT+jUre+Gfy0Y1G3eqxUVLz2+yF0vm/GJZrksJ9811HSIUCKFbFhaRqYdmD5k0sUdouLaOP39+lHxCcnFf/1XhqOywmP/068W1VZVlPo7gwhaChkltygpZPPSHeERXfqOPmekqQCO8IjIiJvvmXho9MSYpDtnfFp+2LXXVBYRdDJz8l2jTIcIFVLIBqVlZIYBlw3NurxzeFRMvMksymZTSWdfOHbnb//ctfOdM/KLd2wtMJlHBJVZpgOECilks0ZFxMR16DF89AjTQY5K7JU2cPejr6UnP/w/W11ffPKZ6TwiKJyTk+8abTpEKJBCNiQtIzMKuGTI+dNSwiIioxt9gh/FJjo7Vj704tkRKxa7iua8kOtxu+tNZxIB73emA4QCKWRzxgKxXQeNGGw6yPGER0bFxtz5wAVERNlKHv3jgrrqqnLTmURAOy8n39XNdIhgJ4VsgHfu+MLeZ4yPiIyLTzKd50Rsdrs9Oevn53omZnWufuD2DytLDu83nUkELBvwC9Mhgp0UshlpQGyfs84OyNHxjzkHDhum73pgeO1f7l5asnv7ZtN5RMC6Liff1aJrsLQ2UshmZMYnp3jaduzaz3SQpopv37F71P1/P6/y9efyD6xd+YXpPCIgtQd+ZjpEMJNC9rO0jMwOQP/B503tbrPZgurvPyImrm37X82aWrO14ODOD+d86PG43aYziYDzS9MBgllQFUKIGK1sdk/HtMFBecup3REW3nnaNVnhySnx2/79+Jy6mupK05lEQBmXk+/qazpEsJJC9qO0jMwI4Oz+GefFm74R5FQopUgZPnp08rmTh2x+9sE5VaUlRaYziYBys+kAwUoK2b8GA5G9Ro4dYjpIS2jbuXu/HtfcMWnTa8+8X1K4Z5vpPCJgXJ2T74oyHSIYSSH7iXeJzUmxick1bdql9jSdp6VExye0G3T9XZdtX/7xl/s2rVtpOo8ICG2B802HCEZSyP7TGejWb8zEFGWzKdNhWpIjPCJqyMVX/7yi5FDJN8v+84H2eDymMwnjLjQdIBhJIfvPEMCd0ndA0Fzq1hzKZlO9R43PTOzYteNXC15/o76uttp0JmHU+Tn5rpAaePiDFLIfeKcrRkfExJW3ad8xZKYrjqddj75D+o2eOHbVO6+8WV1edth0HmFMOyBgFs0KFlLI/pECOPuOPifVZrfbTYfxtdik5E5Ds34+df3H735UerBwh+k8wpiLTAcINlLI/jEAIKXPgN6mg/hLeFR03NCsyy/bU7B23f5tBWtM5xFGyDxyM0kh+8eZQEnb1C4hPV3xYza73Z42blKWu66udtvKvI9lI9VWZ3BOvquT6RDBRArZx9IyMtsAXTqdlh4VHhUdZzqPCR37Dz4juVvvnhsWL3jXXVdXazqP8CsZJTeDFLLv9QToOnhkL9NBTGrTPrVH71Hjx21YsmB+TWV5iek8wm+kkJtBCtn3TgdqEjt162o6iGmRMXFtB5x9UdZ3q1csLzt0cLfpPMIvxufku0L+RHZLkUL2obSMTDuQDhTHJjpTTOcJBHZHWHi/MRMvLN67c0fRjq3rTecRPhcNhOS1974ghexbyUBkQkrn8LDIqFjTYQKFUooug4aPsdkdjl3rVi2Tc30hL910gGAhhexbKYBK7TdQRsfHkdS5e39n1159vv3y0/+46+vrTOcRPnO66QDBQgrZt7oCnqTOPaSQTyC6Tdv23dLPGLP9qxVLa6sqy0znET4hI+QmkkL2rb5AeZt2KVLIP8ERHhHVc/iYc/ZvK/i6oti1z3Qe0eLSZV2LppFC9pG0jEwb0B0oj0lMlkJuhLLZbF0GDhtdWVpyuHjfri2m84gW1QboYTpEMJBC9p0kICw2qV1YRHRMG9NhgkVy114DwiKiovdv2yS3W4cWmbZoAilk30kB6Jg2REbHzRSblNwpsWPX3ns3fb3S43bXm84jWoSc2GsCKWTf6QKQ1Klbe9NBglF4VHRcSp8BQw9s25RfV11VYTqPOGUhsW2Zr0kh+05foDwqPiFoNzM1zWa3O1L6DhheWrR/p2ykGvS6mw4QDKSQfacbUBERHSs3hJyipM7d0+rraqvLXAd2mc4iTpqs+tYEUsg+kJaR6QBigdrw6JhWucJbS4tLatc5PCo6rrhw91bTWcRJic3Jd8nJ7UZIIftGLOABCI+MlhFyC4mIiWsb7+zQ9fDenZs9spFqMJJRciOkkH0jFtAAjshIKeQWZA8LC2+b2qVvWdH+HfW1NVWm84hmSTUdINBJIftGHEB4VIzDERYeYTpMqFFK0aZ9ao+ayoojtVUVpabziCZLNh0g0Ekh+0YsYGvTPlVGxz4Uk5DYQWt0dXmp7G4dHJymAwQ6KWTfiANUXHJ7OaHnYxHRMW0cEZHRcllcUJBCboQUsm8kAXUxCU4ZIfuBIyw8MjI2Pqmq7IhLezyyuHLgkkJuhBSybziB2rDIyHDTQVoLZbPZouLaOGurq0o97nq53TowyU+MjZBC9o1EQHZXNiAiOqaNx+2ukwXvA5L89NIIKWTfiALcyN5ERjjCI6KUQkkpBxz5fmiEFLJvyD88w2x2h8Nmszk8brfbdBZxjHxfNMJhOkCI0sf+I4xRNptSYNdao5RsWBEA5O7KRsgI2ZdkyiIgSBkHDPl+aIQUsg9JHwvxA/IN0QgpZN/QgAzLhPghmbJohBSyb+gf/SqEkO+HRkkh+4Z1Uk/LXWNCNCDfD42QQvahmvKyatMZhAggsghUI6SQfcMDqIqSw7I5pxDf22c6QKCTQvaNMsBRfuigFLIQ35NCboQUsm8UA+GlRfulkIX43l7TAQKdFLJvFANh7rpaj2wzJMQxMkJuhBSyb5QAdgDZYkgIwLrCotB0iEAnhewb5Xgvgq+pKD9iOIsQgeBgdrpT1qluhBSybxzBe81lVdkRKWQhZLqiSaSQfeMI3lunq46USCELIYXcJFLIvlGK9++2tKjwkOEsQgSCnaYDBAMpZB8oyMutxZpHDtu3eb2cyBACvjIdIBhIIfvOPiD60K7vjtTVVFeaDiOEYatNBwgGUsi+8w3eXXYril0yfyZaLa11FbDRdI5gIIXsOzvw/v0eObBPClm0WkqptdnpTtnbsAmkkH1nH95L3w7t3i7zyKI1k+mKJpJC9p0iwA3Y932zTkbIojWTQm4iKWQfKcjL9QDfAXGH9+worauukoWGRGslhdxEUsi+9Q0QC1AuJ/ZEK6S1Lsf6PhBNIIXsWzs4emJv/15ZelC0OkqpNdnpTtnctImkkH3r2Im93eu/2mY4ixAmLDYdIJhIIfuWC6gDwrbnf763trqq3HQgIfzsQ9MBgokUsg95T+x9BSSiNYf3bN9iOpMQ/qK1PoDcMt0sUsi+txqIBNi7ce1mw1mE8Bul1MLsdKc2nSOYSCH73hasxeptWz5f+p27vr7OdCAh/OR90wGCjRSyjxXk5VYAm4GEuurK+iMH9n5nOpMQvqa1rgQ+Mp0j2Egh+8fneBcaKtyyQa7JFK3Bwux0p9wM1UxSyP5xrIS3frZ0i9Za5tVESFNKzTWdIRhJIfuHC+ua5NjSosLK8kNFe0wHEsJXtNY1yPzxSZFC9oOCvFwNrADaAuze8FW+2UTC11bMfpZHLxnD36eNZvlrzwBQuGUDT109iUcvHctLd8ygurzsv55XtGMb/7hs3LGPB8Z0P/b8Rc/8lb9kDjz2uW+WfwzAjrUreezSDJ644lxcu6xTFFVlR3hh5iUY+mFsXna6s9TECwc7h+kArchGYBrAutx56/ucOWGiIzwi0nAm4QP7t21i1bxXmflyLvawcF68dTr9xpzL3Fl3cf5dD9Bj6Fmsfvc1Pn35CSbO/N0PnpvcrRe3v/EJAB63m7+cN5DTxl9w7PNnzbiZsVfd8oPnLH/laWb87QWK9+1m5dv/5oJfzWLJPx9m3PV3opTy+fv9MaXUk35/0RAhI2T/2Q3sB+Jqqyrq92/bJKPkEFW0fQudBw4lPCoau8NB96FnsnHJh7h2bqP76WcC0OuMcWxc/NM/1W/78lOSOnWjbWrnn/w6m8NBXXU1ddVV2B0ODu3eTunBQnoMPavF3lNTuevrCrLTncv9/sIhQgrZT7x37X0IJAJsXLxgtZzbC03te/Zn+5rPqSg5TG1VJZuXL+LIgb2079mfTXkLAVi/aD4lB356val1ufMYlDn1B499/ua/eOzSDN5+4HaqSksAGHfdHcz7v1+zYvazjJp+A/958s+cOzPbN2+uETa741EjLxwipJD9aw3etS0OfPvN4SMH9n1rOpBoee169CHjmtt4YeY0Xrx1Oil9TsNmd3DxHx7j87de4PHLz6amohx7WPgJj1FfV8umT3MZeG7WscdGXnINv52/itveWEqcsz0fPHI/AKl9BzLz5YXc+Ny7HN6zg7jkDmgNs++5gTfv/SVlhw76/D0DeDzuMqXUa355sRAlhexHBXm5lcASoB3At6uWrTKbSPjK8ClXcNvsJdz0rwVExyeQ1KUH7br35vqn5nDb7MUMPm8qSZ26nfD5W1YsJrXfIOKS2h17LC6pHTa7HZvNxoipV7Jn4w9nvbTWLPnX3zn7xl+z+LmHOOfmexhy/jQ+e/2fvnqbP6R5MTvdKTusnwIpZP9bhnUyVRUs/WBLTWXFEdOBRMsrP1wEQEnhHjYu/YAh50099pjH42Hp848w8uKrT/j8rxe+w+DMn/3gsdKi/cd+v3HJh7Tv2e8Hn1+z4A36jT6HqPgE6qqrUDYbymajrrqqpd7WCWmttc1uf9znLxTi5CoLPyvIy92blpG5GeioPZ5Dezfmr+4xfPTZpnOJlvXab66l8kgxNkcYWfc8SFR8AitmP8vnb70AwIAJFzB08uWAVbRzZ93JtY+/AUBtVSVbV+bxs3sf/sExP3psFoVbNqBQtE3tzJR7/3bsc7VVlax5/02ue3IOAKNn3Mxrv70WuyOMy/7ynM/fr8ddv+Te4Smy5vcpUnJiyf/SMjKHAHcAO+Oc7aMn/+6hu2x2u/zPUQSzrOx05wLTIYKdTFmYsREoA6LKXAcqCzevl00gRdCqr63dgNyZ1yKkkA0oyMutAxYCyQCr5r26TJblFMHK5nDcKesetwwpZHOWYV0CF1FaVFi5t2DtStOBhGiumoryL34/tJ3sm9dCpJANKcjLLQPmA+0BVs17ZUV9XW2N2VRCNJ3WGnt4+C2Nf6VoKilksz4BaoDIimJX9c61K5cZziNEk9VUlH9w34jUNaZzhBIpZIO8u4m8g3eU/OXcl1fKdckiGHg8bndYZORtpnOEGilk85YBpUBMXXVl/eblH8t8nAh4tRUVr9w7PGW76RyhRgrZsIK83Grgdby3U6/96O315cWufWZTCXFi7vr66si4+LtN5whFUsiBYRWwE0hEa9bMf/0D7fHIZUQiINVWlj+Yne4sMp0jFEkhB4CCvFw3MBtoA6gd+V/s271xzQrDsYT4L1VlR7ZGxSf8r+kcoUoKOXBsBlYCKQDLX3nqk8rSEv+smyhEE7jr6+uP7N97WXa60206S6iSQg4Q3n33ZmPdLBJTX1vjXjX3pXe1x+MxHE0IAIr37Xri0UvHymVuPiSFHEAK8nJLgBewLoNTO7/+snDX+q9k6kIYV1Hs+rb8cNGvTecIdVLIgWcN8DmQCrDitaf30AniAAAMnklEQVTzKo8Uy9SFMMZdX1dfWrR/2jPXXiA/rfmYFHKAaTB1UYt36uLLuS/Nk6kLYUrx3l2PPDZ93FrTOVoDKeQAVJCXewR4Hu/Uxa51q/bvXLdKbqsWfld+2LVl76a1ZnZMbYWkkAPXWmAF0BFgxWvPfFpatH+n2UiiNamrqa4+cmDv1Nd/d5NcE+8nUsgByjt18TpQBcS662o9i5/961s1leUlhqOJVkB7PHrPxvzbH7/87I2ms7QmUsgBrCAvtxR4Fmsh+7Ay14HKFa8987q7vq7WcDQR4nZt+OqtZS8/8bzpHK2NFHKAK8jL3QC8AXQG1J6N+Qe/XvjOPNkLUfhK0c5tG/JeePQa709pwo+kkINDLtaqcJ0BNiya/82ONZ8vMRtJhKIy14GDGxYtmORd9Er4mRRyECjIy/UALwM7gA4Ay155cplr57cbTOYSoaW6vLRiw+IFF+c+8ac9prO0VlLIQaIgL7cGeAKoBhIAFj3z4HsVxYcKjQYTIaG+tqZu/aL5t7+Xk73cdJbWTAo5iBTk5R4GHgPigKjaqor6Jc8//LrsMiJOhcft9mzKW/jIpk8+etFXr6GUukYpldoSz1VK3amUiv6Jr39eKZXm/X15M19riFLq/JPI+IlSalhzn/djUshBpiAvdzvwT6xV4ezFe3eWLf3n316SUhYnw+N2e9Yveu+F/A/eus/HJ/GuwbscQAs8907guIWslLJrrW/QWhec5GsNAZpdyC1FCjk4rQTeA7oCtoPbtxRLKYvm8rjdnvwP3nr764/m3lGQl1vfnOcqpboppTYppf6plNqolPqPUirKO8L8Qim1Tik1TynVVik1DRgGvKaUWquUijrBMe9XSq1SSm1QSj2nLD9+7h1Y5bxUKbXU+7xypdQspdRKYNSPR6tKqYeVUmuUUouVUsnex459jVLKqZTaoZQKB2YB072vNV0pFaOUesGbK18pNdn7nCil1Bve9/kmcNz31FxSyEHIO5KZBywEuuEt5SXPPfRvKWXRFB6327NmwRtzNy55/6aCvNzKkzxMb+BJrfVpQAlwMdbJ53u01oOA9cAftNZvA6uBGVrrIVrrqhMc7wmt9XCt9QCsgrvwOM99DNgHjNdaj/c+LwbYoLUeqbX+8Rx4DLBGa306kAf84URvRmtdC9wPvOl9rTeBe4ElWuvhwHjgIaVUDPBLoNL7Pv8EDG3qX9pPkUIOUt4rL96gQSkX7dhaIqUsGuNx17u/mv/62wWffPgL75KvJ2u71vrookNfAT2BBK11nvexl4CxzTjeeKXUSqXUemACcFoTn+cG5p7gcx7gTe/vXwVGNyMPwEQgWym1FvgEiAS6YL2vVwG01uuAdc087nFJIQcxKWXRXEfLeFPeRzedYhkD1DT4vRvv1T8nQykVCTwFTNNaD8Q6TxLZxKdXa62buovJ0Xnyer7vv596HQVc7B0xD9Fad9Fab/rRsVqMFHKQk1IWTeVx17u/em/2nE15C1uijI/nCFCslBrj/fOVWNMEAGVYVwedyNFSdCmlYoFpDT734+c2dqyGbA2OdTlwdEpjB99PM/zUa+UCtymlFIBSKt37+KfADO9jA4BBTczTaFgR5BqU8kc0KOXFz/3131VlR1xGw4mAUF9bW7vq3Vff3PRp7s3e5V195WqsedZ1WFcszPI+/m/gmROd1NNal2CNitcD72LtxM4Jnvsc8NHRk3qNqABOU0p9hTUNcjTP34BfKqU+A5wNvn4pkHb0pB7wv0AYsE4ptcH7Z4CngVjv+7wb+LIJWRqlZE2E0JGWkWkDpgOTgJ2AO7pNYsS5M393aZv2qT3MphOmVJWWHFn26lOz92/ZmO1dsEoEKCnkEOMt5UuAC4A9QK3NEWY756bfnt+h92ktciZYBI9De3bsXfr8I69Wlhz6s5Rx4JNCDkFpGZkKGId1QX0RUA5wxvTrR/YeOW6istlkqqoV2L7m84IVs595ylNf/0JBXu6JLjXzO6XUPKD7jx6+R2udayJPIJFCDmFpGZmnAXdgnQ0/BNB71IRuw6bMmBYWERljNJzwGY/b7V770dtfbFg0/0HgA+85BhEEpJBDXFpGZiesW03bAHsBEjt1ix93/V3TY9s6T/ZWVhGgaisrKpbPfmbRng1r/liQl5tvOo9oHinkViAtIzMeuBEYCOwG3GERkfaM6+6cmNJnwAjvFT0iyJUU7in89KXH3yvZv+fPBXm5u03nEc0nhdxKpGVkOoDJ3o/9QCVYUxinXzh9ckRM7Elf1C/MctfX1W3KW5i/5v0356L1M3LyLnhJIbcyaRmZQ4FfeP9YCBAZGx8+5spbzunQ57ThMloOLkcO7N396ctPri7eu/NtYE5BXm6d6Uzi5Ekht0JpGZlO4CqsC/f3YS16T58zJ3RLv0BGy8HAOypetWbBm6tBPw9skD3wgp8UcivlvV75TKzbW0FGy0GjZP/e3cteeXJ18d6dC7BGxc1ahF0ELinkVu6nRstDzr/0osjYuEST+cT33HV1tZvyPlq95v23VnlHxRtlVBxapJDFCUfL9rBw27DJlw/tMWz02LDIqFhjAVs57fF49m1ev37lnBc3lx8uWoiMikOWFLI4xjtavhoYDLiAUoCImLiwEVOvOqPLoOFn2cPCIkxmbG0O7925ZeWcF9cX7di6G3geKJBRceiSQhY/4B0tDwEuA9oBB/FeIheb1C5qxMVXj0ntO3CEzW63G4wZ8koPFu7M/3DO1zvXrizCWsVvYUFeboXpXMK3pJDFcaVlZIYBZ2AtVBSHde1yDUBS5+7xw6ZcMb5d9z6Dlc0mZ/5aUNmhg7u/XvhO/nerlhUDK4B3C/Jyi0znEv4hhSx+UlpGZiTWQkVTsNaFLcTabYGUPgOcA865aGS7Hn0H2R1h4eZSBjettS4p3LPtm09zN239YmkJsBFrnni76WzCv6SQRZOkZWTGYe0vNglr65r9WNv2EBXXJnxQ5tTBXQYNGx4Vn5BsMGZQqauprizcsmHNutx53x7eswNgM9becFtlnrh1kkIWzZKWkZkEnA9kYO04cwhrVwYAep8xvmvvMycMT+zUrb9Nlvk8rjLXgV3bv/pszfpF8w+562rtwDbgHeAbKeLWTQpZnBTviHk4VjknAVVYV2Z4ABI6dIodOHHK6R37Dx4aHhUdby5pYHDX19Ue/G7Luo1L3i/Y9806D1CHtd/cp8AeKWIBUsjiFKVlZNqBPsA5QDrWdMZBvCcAlc2megw9q1PX9DP6J3fr1T8iuvXcll1bXVV+eM/2LXs2rt269fMl5XXVVXasOfgPgXy5akL8mBSyaDHe65hHAZlADNao+RDeuWaALoOHp3RLH9XX2bVn75iEpNRQuz27suTw/qId27bsWPvF5p1rv6wE3Qbrf1JfYG2guU1Gw+JEpJBFi0vLyAwHTgNGAqcDDqAWq5yPrUYWn5wS3XPk2F4deqX1ik/u0DkYFzWqr62pKj9UtPfAt5u2bP3ik28P79nh4Pst7b8FPgfWFOTllphLKYKFFLLwKe9lc72x5puHA0cvjysByhp+bWxiclTHtCEpzq49UxLad0yNTUpOiYiJa+vfxCdWV1NdWVF8qLD0YGHh4T3b9xVu2VhYtGNrDZCIdYKzHsgHVgNbZF1i0VxSyMJvvIvkdwH6Y42eO2GdBFRYV2qUY42kj4lp64zsmDYkJblrr9S45PbJEdGxceFR0XFhkVFxjvCISFqY1pq6mury2sqKIzUVZaXlh4qKDu3ZXli4ef2+Q7u3lwOxWDfKHL1T8QjWKPhrYHtBXm7tCQ4tRKOkkIUxaRmZbbBKuTPQF2skHdXgSyqxRtHHLbmwyGhHQoeOsXHJ7eNiE5Pjotu0jY2MbRNnczj+67ZuZZX+MR6Px1NbWVFZXVFaUV1WWll5pLiy7NCB8uK9u0rddbUerMKN9X7YseaBPcAuYAvWdMQ+oFDmhEVLkUIWASMtI1MBCUAqVlH3A3phnSA8unOy4vvpgRqssj76a1P/MYf96CPc+6vHewyFdSJyB9bNGjuwyreoIC+3/uTfoRA/TQpZBDRvScdgTRPE8f2UgbPBRyLWrtqO4xzi6D9w1eDPFVgr2R3Bmss+DBxo8FgpUFKQl+tGCD+SQhYhwVvcJ7qGruHjuiAv13OCrxPCKClkIYQIELLWgBBCBAgpZCGECBBSyEIIESCkkIUQIkBIIQshRICQQhZCiAAhhSyEEAFCClkIIQKEFLIQQgQIKWQhhAgQUshCCBEgpJCFECJASCELIUSAkEIWQogAIYUshBABQgpZCCEChBSyEEIEiP8HfvYYOpJJsv0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# visualize the imbalance: extremely imbalanced\n",
    "labels = 'attributed', 'not_attributed'\n",
    "sizes = [num_attributed_time[1], num_attributed_time[0]]\n",
    "colors = ['red', 'skyblue']\n",
    "explode = (0.5, 0)  # explode 1st slice\n",
    " \n",
    "# Plot\n",
    "plt.pie(sizes, explode=explode, labels=labels, colors=colors,\n",
    "        autopct='%1.2f%%', shadow=True, startangle=140)\n",
    " \n",
    "plt.axis('equal')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocess Raw Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ip</th>\n",
       "      <th>app</th>\n",
       "      <th>device</th>\n",
       "      <th>os</th>\n",
       "      <th>channel</th>\n",
       "      <th>click_time</th>\n",
       "      <th>attributed_time</th>\n",
       "      <th>is_attributed</th>\n",
       "      <th>day</th>\n",
       "      <th>hour</th>\n",
       "      <th>minute</th>\n",
       "      <th>second</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>83230</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>379</td>\n",
       "      <td>2017-11-06 14:32:21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>14</td>\n",
       "      <td>32</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>17357</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>379</td>\n",
       "      <td>2017-11-06 14:33:34</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>14</td>\n",
       "      <td>33</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>35810</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>379</td>\n",
       "      <td>2017-11-06 14:34:12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>14</td>\n",
       "      <td>34</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>45745</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>478</td>\n",
       "      <td>2017-11-06 14:34:52</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>14</td>\n",
       "      <td>34</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>161007</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>379</td>\n",
       "      <td>2017-11-06 14:35:08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>14</td>\n",
       "      <td>35</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       ip  app  device  os  channel          click_time attributed_time  \\\n",
       "0   83230    3       1  13      379 2017-11-06 14:32:21             NaN   \n",
       "1   17357    3       1  19      379 2017-11-06 14:33:34             NaN   \n",
       "2   35810    3       1  13      379 2017-11-06 14:34:12             NaN   \n",
       "3   45745   14       1  13      478 2017-11-06 14:34:52             NaN   \n",
       "4  161007    3       1  13      379 2017-11-06 14:35:08             NaN   \n",
       "\n",
       "   is_attributed  day  hour  minute  second  \n",
       "0              0    6    14      32      21  \n",
       "1              0    6    14      33      34  \n",
       "2              0    6    14      34      12  \n",
       "3              0    6    14      34      52  \n",
       "4              0    6    14      35       8  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# transform click_time into day, hour, minute and second\n",
    "talking['day'] = talking['click_time'].dt.day\n",
    "talking['hour'] = talking['click_time'].dt.hour\n",
    "talking['minute'] = talking['click_time'].dt.minute\n",
    "talking['second'] = talking['click_time'].dt.second\n",
    "talking.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['ip', 'app', 'device', 'os', 'channel', 'click_time', 'attributed_time',\n",
      "       'is_attributed', 'day', 'hour', 'minute', 'second'],\n",
      "      dtype='object')\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 184903890 entries, 0 to 184903889\n",
      "Data columns (total 12 columns):\n",
      "ip                 category\n",
      "app                category\n",
      "device             category\n",
      "os                 category\n",
      "channel            category\n",
      "click_time         datetime64[ns]\n",
      "attributed_time    object\n",
      "is_attributed      category\n",
      "day                category\n",
      "hour               category\n",
      "minute             category\n",
      "second             category\n",
      "dtypes: category(10), datetime64[ns](1), object(1)\n",
      "memory usage: 5.7+ GB\n"
     ]
    }
   ],
   "source": [
    "# ip, app, device, os, channel, is_attributed and the time features we just created\n",
    "# are encoded to be integers, we tranform them into categorical data\n",
    "heads = talking.columns\n",
    "print(heads)\n",
    "for i in list(range(0,5)) + list(range(-1, -6, -1)):\n",
    "    talking[heads[i]] = talking[heads[i]].astype('category')\n",
    "talking.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAEICAYAAABcVE8dAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3XmYVNWd//H3h0UR2ZQtjYiggiIqPYCixgUkxCUKEUiQIYmJZhyT0RgdE82MmewJOmNcokkeJ5gQQ0DBBSZoNKBIiFEDsgQ0Kj9BUZBFWUWkG76/P+7ttmm6oaup6gLu5/U89XTd9XzPrepvnTr31rmKCMzM7MDXqNgBmJlZw3DCNzPLCCd8M7OMcMI3M8sIJ3wzs4xwwjczywgn/P2MpF9K+nax49hfSDpO0jxJmyR9rcBlPS7pskKWUUu5fk9YnTjhNzBJIenYavO+K+l3ddk+Iq6KiB8UJrrdk3SepFlp8lwj6RlJQxqg3GWSPlHPzb8JzIyIlhFxVx5i+a6kMkmbqzy+CRARF0TEuHrud5f3RbXlX5S0vVq5d6flFuQ9kR73D6qV2Wkv9zlA0lv5itFy06TYAdj+QdII4D7geuBiYBNwFvA5YGoRQ9uTo4CJ9dlQUpOIKK9h0QMR8bk87SsXf42IM/dyH7m6OCKmN3CZtcrTccwst/D3MRUtIEn/Lmm1pJWSvlRl+W8k/bDK9DfSdVZIurxqS1HSTElfrrLuFyXNrjJ9vKQ/SXpP0iuSPltLTAJ+CvwgIn4VERsiYkdEPBMR/5Ku00jSzZLeSOP+raTWVetUbZ+Vrfa01fxgus0mSYsl9UuX3Q90Af6vojUtqZmk30l6V9J6SX+T1LGGuJ8CBgJ3p9v2kNQ6LWdNGuvNkhpVOT5/kXS7pPeA7+b42lUe75r2JenY9FvRBklrJT2Qrjsr3cWCNM6ROZZb+Z6Q9LKki6osa5KW1SedPk3Ss+lxWyBpQC5lVdlvrfuR9KU0jk2SXpf0r+n8Q4HHgU5VvzHU8J7e6f2SvldulLQQeD+tUydJD6Wv41JV6a6TdKqkOZI2Slol6af1qeOByAl/3/QxoDVwBHAFcI+kw6qvJOl84AZgMNAdqHO3R/rP9yfg90AHYBTwc0m9alj9OOBIYPJudvnF9DEQOBpoAdxd13iAISQt8TYk3xgquis+D7xJ0tJsERG3ApeRHJ8jgbbAVcAH1XcYEecCfwauTrd9FfhZuu3RwDnAF4AvVdmsP/A6yTH5UQ7x16T6vn4APAkcBnROYyEizk7X753G+cBelDmB5LWscB6wNiJelHQEMA34IXA4yXvnIUntcymgDvtZDVwEtCI5trdL6hMR7wMXACvSeraIiBV1LHYU8CmS98cO4P+ABST/I4OAr0s6L133TuDOiGgFHAM8mEv9DmRO+PumMuD7EVEWEY8Bm0mSbnWfBX4dEYvSf6bv5lDGRcCyiPh1RJRHxIvAQ8CIGtZtm/5duZv9jQZ+GhGvR8Rm4FvApZLq2m04OyIei4jtwP1A792sW5bGdGxEbI+IuRGxcU8FSGoMjAS+FRGbImIZcBvw+SqrrYiIn6XHZJcPkdRn05ZtxaO2fu3q+yoj6WLqFBFbI2J2LdvV5rRq5Z5Wwzq/B4ZIap5O/3M6D5Lut8fS47wjIv4EzAEu3E2Zj1Yp79G67CcipkXE/4vEMyQfcmflWNfq7oqI5elxPAVoHxHfj4htEfE68L/Apem6ZcCxktpFxOaIeG4vyz5gOOE3vO1A02rzmpK8SSu8W62fcgtJi7m6TsDyKtNv5BDHUUD/qgmEJGl/rIZ1303/luxmf52qlf8GyTmiXbpaavFOledbgGa7+bC4H3gCmKikK+tWSdWPaU3aAQfVEOcRVaaXs2cPRkSbKo/aWqnV9/VNQMALabfV5XUoq6rnqpW7SyKLiCXAy8DFadIfwkcJ/yjgM9Ve8zPZ/ev66Srlfbou+5F0gaTnlHQVrif5IGiXY12rq3osjyLpFqpa/n/w0XvtCqAH8I+0u+8iDPBJ22J4E+hK8k9ZoRvwaj32tZKkW6NCl2rL3weaV5mumsyXA89ExOA6lPNKuv5w4H9qWWcFyT9i1VjKgVUkHwaVcaQt7Vy6EXYa0jUiyoDvAd+T1BV4LI1x7B72s5aPWtkvVYnz7drK2kvV434HqDjncSYwXdKsNEnnU0W3TiPgpSr7Xw7cX3HeZS/Uuh9JB5N8U/wCMCUiytJvBkpXqen47u59WqHqdsuBpRHRvabgIuI1YFR6bmYYMFlS2/RbcKa5hd/wHgBultRZyYnOT5Bc9bK7/vHaPAh8UdIJaWvuO9WWzweGSWqu5ETuFVWW/QHoIenzkpqmj1Mk9axeSCRjaF8PfDs9Idcqjf1MSfemq00ArpPUTVIL4MckV7OUk3yYNZP0qbQlfjNwcA71XEXS5w6ApIGSTko/ODaSJPHte9pJ2l30IPAjSS0lHZXWq06XxO4tSZ+R1DmdXEeSxCri3qmOe2ki8EngK3zUuoeknhcruby2sZKT3wOqxFRXu9vPQSSv7RqgXNIFaSwVVgFtlZ7QT80HLpR0uKSPAV/fQ/kvABvTE7mHpDGcKOkUAEmfk9Q+InYA69Nt9vj+yAIn/Ib3feBZYDbJP/2twOiIWJTrjiLiceAO4ClgSfq3qtuBbST/ZOOA8VW23UTyj3gpSev8HeAWaknEETGZpP/78nT9VSQn7aakq9xH0tUyC1gKbAWuSbfdAHwV+BVJa/p9IJdrsX9C8iG5XtINJC3AySTJ/mXgGeqetK9Jy3+d5DX4fRp7QzgFeF7SZpIT09dGxNJ02XeBcWkda7xaqq4iYiXwV+AMkgZGxfzlwFCS7o81JC3lb5BjHtjdftL31ddIPljXkZxDmFpl23+QNA5er3L+436SE7DLSPr7d3vSOv3gvhgoJXmvrSV5b1V8iJwPLE6P853ApRGxNZc6HqjkG6AcWCQF0L0A3QRmtp9zC9/MLCOc8M3MMsJdOmZmGeEWvplZRuxT1+G3a9cuunbtWuwwzMz2K3Pnzl0bEXv8bcs+lfC7du3KnDlzih2Gmdl+RVKdfmXvLh0zs4xwwjczy4jMJ/zly5czcOBAevbsSa9evbjzzjsBGDlyJKWlpZSWltK1a1dKS0sBGD9+fOX80tJSGjVqxPz58wE4//zz6d27N7169eKqq65i+/bk19wLFizg9NNP56STTuLiiy9m48aPBnZcuHAhp59+Or169eKkk05i61b/INDMCiQi9plH3759o6GtWLEi5s6dGxERGzdujO7du8fixYt3Wuf666+P733ve7tsu3DhwujWrVvl9IYNGyIiYseOHTFs2LCYMGFCRET069cvZs6cGRERY8eOjZtvvjkiIsrKyuKkk06K+fPnR0TE2rVro7y8PM81NLMDHTAn6pBjM9/CLykpoU+fPgC0bNmSnj178vbbHw2eGBE8+OCDjBo1apdtJ0yYsNP8Vq1aAVBeXs62bduQkgECX3nlFc4+O7nHxeDBg3nooYcAePLJJzn55JPp3TsZ+r1t27Y0bty4ALU0M3OXzk6WLVvGvHnz6N+/f+W8P//5z3Ts2JHu3XcdifWBBx7Y5YPgvPPOo0OHDrRs2ZIRI5J7iZx44olMnZqMHzVp0iSWL0+G9n711VeRxHnnnUefPn249dZbC1U1MzMn/AqbN29m+PDh3HHHHZUtddi1FV/h+eefp3nz5px44ok7zX/iiSdYuXIlH374IU89lQxeed9993HPPffQt29fNm3axEEHHQQk3wRmz57N+PHjmT17No888ggzZswoYC3NLMuc8IGysjKGDx/O6NGjGTZsWOX88vJyHn74YUaO3PWe0hMnTqzxgwCgWbNmDBkyhClTkpGDjz/+eJ588knmzp3LqFGjOOaYYwDo3Lkz55xzDu3ataN58+ZceOGFvPjiiwWooZmZEz4RwRVXXEHPnj25/vrrd1o2ffp0jj/+eDp33vn+EDt27GDSpElceumllfM2b97MypXJLV/Ly8t57LHHOP744wFYvXp15XY//OEPueqqq4Ck+2fhwoVs2bKF8vJynnnmGU444YSC1dXMsm2fGjytX79+sTe/tO1607Sct9n61mJWjb+Rpu27QnqS9bCzv8Ahx5zC2mm3c3Cn42j5Tzvf43nrmwtZN3McJV+4rXLe9vfXsXry94ntZbBjB82OOpnDBv0LatSYjXOmsOnFJLbmPc6gzTmXVZ7Q3bz4aTb+dRIIDjm6H4cNvJxlYz5VzyNgZlkkaW5E9NvjellP+PsiJ3wzy0VdE37mu3TMzLLCCd/MLCOc8M3MMsIJ38wsI5zwzcwywgnfzCwjnPDNzDLCCd/MLCOc8M3MMsIJ38wsI5zwzcwyokmhC5C0DNgEbAfK6zLeg5mZ5V/BE35qYESsbaCyzMysBu7SMTPLiIZI+AE8KWmupCsboDwzM6tBQ3TpfDwiVkjqAPxJ0j8iYlbFwvRD4EqALl26NEA4ZmbZVPAWfkSsSP+uBh4BTq22/N6I6BcR/dq3b1/ocMzMMqugCV/SoZJaVjwHPgksKmSZZmZWs0J36XQEHknv39oE+H1E/LHAZZqZWQ0KmvAj4nWgdyHLMDOzuvFlmWZmGeGEb2aWEU74ZmYZ4YRvZpYRTvhmZhnhhG9mlhFO+GZmGeGEb2aWEU74ZmYZ4YRvZpYRTvhmZhnhhG9mlhFO+GZmGeGEb2aWEU74ZmYZ4YRvZpYRTvhmZhnhhG9mlhFO+GZmGeGEb2aWEU74ZmYZ4YRvZpYRTvhmZhnhhG9mlhFO+GZmGeGEb2aWEU74ZmYZ4YRvZpYRTvhmZhnRIAlfUmNJ8yT9oSHKMzOzXTVUC/9a4OUGKsvMzGpQ8IQvqTPwKeBXhS7LzMxq1xAt/DuAbwI7GqAsMzOrRUETvqSLgNURMXc361wpaY6kOWvWrClkOGZmmdakLiul3TKXAmcBnYAPgEXANODxiKit9f5xYIikC4FmQCtJv4uIz1WsEBH3AvcC9OvXL+pbETMz2709tvAl/Rq4D9gG3AKMAr4KTAfOB2ZLOrumbSPiWxHROSK6knxgPFU12ZuZWcOpSwv/tohYVMP8RcDDkg4CuuQ3LDMzy7c9tvArkr2ka6svk3RtRGyLiCV12M/MiLiofmGamdneyuWk7WU1zPtinuIwM7MC22OXjqRRwD8D3SRNrbKoJfBuoQIzM7P8qksf/rPASqAdcFuV+ZuAhYUIyszM8m+PCT8i3gDeAE4vfDhmZlYode7DlzRM0muSNkjaKGmTpI2FDM7MzPInl5O2twJDIqJ1RLSKiJYR0apQgZkV2tatWzn11FPp3bs3vXr14jvf+c5Oy6+55hpatGhROX3ddddRWlpKaWkpPXr0oE2bNpXLGjduXLlsyJAhDVYHs1zU6Ze2qVUR4REv7YBx8MEH89RTT9GiRQvKyso488wzueCCCzjttNOYM2cO69ev32n922+/vfL5z372M+bNm1c5fcghhzB//vwGi92sPnJp4c+R9ICkUWn3zjBJwwoWmVmBSapswZeVlVFWVoYktm/fzje+8Q1uvfXWWredMGECo0aNaqhQzfIil4TfCtgCfBK4OH34h1S2X9u+fTulpaV06NCBwYMH079/f+6++26GDBlCSUlJjdu88cYbLF26lHPPPbdy3tatW+nXrx+nnXYajz76aEOFb5aTOnfpRMSXChmIWTE0btyY+fPns379ei655BJmzZrFpEmTmDlzZq3bTJw4kREjRtC4cePKeW+++SadOnXi9ddf59xzz+Wkk07imGOOaYAamNVdnRN+OojaLqNZRsTleY3IrAjatGnDgAEDePrpp1myZAnHHnssAFu2bOHYY49lyZKPRg+ZOHEi99xzz07bd+rUCYCjjz6aAQMGMG/ePCd82+fk0qXzB5LhkKcBM0i6eDYXIiizhrBmzZrKE7MffPAB06dPp2/fvrzzzjssW7aMZcuW0bx5852S/SuvvMK6des4/fSPfpaybt06PvzwQwDWrl3LX/7yF0444YSGrYxZHeTSpfNQ1WlJE0iGSDYruq43Tct5m22rl7J22u0QOyB20Pz4s7h6trh69kf72rJt+077Xj97PPGxU+j2rccq521962Xee+JukCCClv2GcuFvlwJLc45p2ZhP5byNWV3lcllmdd3xsMi2HzuoQzc6femu3a7T5frJO023OXP0Lus069yTTlfcs8t8s31NLn34m0j68JX+fQe4sUBxmZlZnuXSpdOykIGYmVlh5dSlI2kIUHE7w5kR8Yf8h2RmZoWQy+BpY4BrgZfSx7WSflKowMzMLL9yaeFfCJRGxA4ASeOAecC3ChGYmZnlVy7X4QO0qfK8dT4DMTOzwsqlhf8TYJ6kp0mu1Dkbt+7NzPYbuVylM0HSTOAUkoR/Y0S8U6jAzMwsv3I5aXsJsCUipkbEFGCrpE8XLjQzM8unXPrwvxMRGyomImI98J3drG9mZvuQXBJ+TevuzdAMZmbWgHK949VPJR0j6WhJtwNzCxWYmZnlVy4J/xpgG/AA8CDwAfBvhQjKzMzyL5erdN4HbqptuaSfRcQ1eYnKzMzyLtcfXu3Ox/O4LzMzy7N8Jnwz24/cfvvt9OrVixNPPJFRo0axdetWli5dSv/+/enevTsjR45k27ZtAHz44YeMHDmSY489lv79+7Ns2bLiBm/1UtCEL6mZpBckLZC0WNL3ClmemdXN22+/zV133cWcOXNYtGgR27dvZ+LEidx4441cd911vPbaaxx22GGMHTsWgLFjx3LYYYexZMkSrrvuOm680bfC2B/lM+GrhnkfAudGRG+gFDhf0ml5LNP20iuvvEJpaWnlo1WrVtxxxx289957DB48mO7duzN48GDWrVtXuc3MmTMpLS2lV69enHPOOUWM3vZGeXk5H3zwAeXl5WzZsoWSkhKeeuopRowYAcBll13Go48+CsCUKVO47LLLABgxYgQzZswgIooWu9VPzglf0qG1LLqz+oxIVNzovGn68LtkH3Lccccxf/585s+fz9y5c2nevDmXXHIJY8aMYdCgQbz22msMGjSIMWPGALB+/Xq++tWvMnXqVBYvXsykSZOKXAOrjyOOOIIbbriBLl26UFJSQuvWrenbty9t2rShSZPkWo7OnTvz9ttvA8k3giOPPBKAJk2a0Lp1a959992ixW/1k8vQCmdIegl4OZ3uLennFcsj4je1bNdY0nxgNfCniHi+2vIrJc2RNGfNmjX1qYPlyYwZMzjmmGM46qijdmrRVW3p/f73v2fYsGF06ZLczrhDhw5Fi9fqb926dUyZMoWlS5eyYsUK3n//fR5//PFd1pOSL+41teYrltn+I5cW/u3AecC7ABGxgI/uflWriNgeEaVAZ+BUSSdWW35vRPSLiH7t27fPIRzLt4kTJzJq1CgAVq1aRUlJCQAlJSWsXr0agFdffZV169YxYMAA+vbty29/+9uixWv1N336dLp160b79u1p2rQpw4YN49lnn2X9+vWUl5cD8NZbb9GpUycgae0vX74cSLqCNmzYwOGHH160+K1+curSiYjl1WZtz2Hb9cBM4PxcyrSGsW3bNqZOncpnPvOZ3a5XXl7O3LlzmTZtGk888QQ/+MEPePXVVxsoSsuXLl268Nxzz7FlyxYighkzZnDCCScwcOBAJk+eDMC4ceMYOnQoAEOGDGHcuHEATJ48mXPPPdct/P1QLmPhLJd0BhCSDgK+Rtq9UxtJ7YGyiFgv6RDgE8At9Y7WCubxxx+nT58+dOzYEYCOHTuycuVKSkpKWLlyZWXXTefOnWnXrh2HHnoohx56KGeffTYLFiygR48exQw/07reNK1e261v05vWRx6HGjXioI7HMLP9JZS3PZ+p1/8Xn//K9RzU8WjatTifW2+aRpR3Yu3MCYw9rBONDmlBuyE31rvc3Vk25lN536d9JJeEfxXJidkjgLeAJ9nz0AolwDhJjUm+TTzoG5/vmyZMmFDZnQMftehuuummnVp6Q4cO5eqrr6a8vJxt27bx/PPPc9111xUrbNsLbc4aTZuzRu80r2mbj1Hyhdt3WVdNDqL9p32/o/1dLkMrrAVG73HFnbdZCPxTrkFZ7vamtbWjbCtvT3mMZ0uG8b10P9u39WHtL8fw7Vvvokmr9rQb+i0eTJdt2HEUzT92NEi0OPk8LvrdG8Ab+agG4FaeWaHUOeFL+jU1XFIZEZfnNSJrcI2aNuPIayfsNK/xIa3oeOmPa1y/df/htO4/vCFCM7M8yqVLp2pXTDPgEmBFfsMxM7NCyaVL56Gq05ImANPzHpGZmRXE3gyt0B3okq9AzMyssHLpw99E0oev9O87gEdQMjPbT+TSpdOykIGYmVlh5dLC77O75RHx4t6HY2ZmhZLLVTo/B/oAC0m6dU4GngfKSLp4zs17dGZmlje5nLRdBvRNBzrrS/KDqiURMTAinOzNzPZxuST84yPi7xUTEbGI5KYmZma2H8ilS+dlSb8CfkfShfM59jB4mpmZ7TtySfhfAr4CXJtOzwJ+kfeIzMysIHK5LHMryU1Qdh1Kz8zM9nl77MOX9GD69++SFlZ/FD5EM7P8Wr58OQMHDqRnz5706tWLO+9Mbsn93nvvMXjwYLp3787gwYNZt24dAOPHj+fkk0/m5JNP5owzzmDBggXFDL/e6nLStqIL5yLg4hoeZmb7lSZNmnDbbbfx8ssv89xzz3HPPffw0ksvMWbMGAYNGsRrr73GoEGDGDNmDADdunXjmWeeYeHChXz729/myiuvLHIN6mePXToRsTL9m78Bz83MiqikpKTyns0tW7akZ8+evP3220yZMoWZM2cCcNlllzFgwABuueUWzjjjjMptTzvtNN56661ihL3X6nxZpqRhkl6TtEHSRkmbJG0sZHBmZoW2bNky5s2bR//+/Vm1alXlB0FJSQmrV6/eZf2xY8dywQUXNHSYeZHLVTq3AhdHhC/FNLMDwubNmxk+fDh33HEHrVq12uP6Tz/9NGPHjmX27NkNEF3+5fLDq1VO9mZ2oCgrK2P48OGMHj2aYcOGAdCxY0dWrlwJwMqVK+nQoUPl+gsXLuTLX/4yU6ZMoW3btkWJeW/lkvDnSHpA0qi0e2eYpGEFi8zMrEAigiuuuIKePXty/fXXV84fMmQI48aNA2DcuHEMHToUgDfffJNhw4Zx//3306NHj6LEnA+5dOm0ArYAn6wyL4CH8xqRmVkddb1pWr222/rWYlaNv5+m7bvy8wlTATjs7C9wUKc+rP3lGL596100adWedkO/xYM3TePdx+9iy1vvMOCSzwGgRo0pueyOvNUDYNmYT+V1fzXJ5YdXXypkIGZmDaVZ514cdeMfalzW8dIf7zKv7QVfo+0FXyt0WAWXy3j4vyZp0e8kIi7Pa0RmZlYQuXTpVP04bAZcAqzIbzhmZlYouXTpPFR1WtIEYHreIzIzs4LI5Sqd6roDXfIViJmZFVYuffib2LkP/x3gxrxHZGZmBZFLl07LQgZiZmaFVZfhkbvuYbkkdc5XQGZmVhh1aeH/t6RGwBRgLrCG5CqdY4GBwCDgO8Auw8dJOhL4LfAxYAdwb0TcmZ/QzcwsF3UZHvkzkk4ARgOXAyXAByT3s50G/Ci9G1ZNyoF/j4gXJbUE5kr6U0S8lJ/wzcysrurUh58m6P/MdefpWPoV4+lvkvQycATghG9m1sByuUqnpoHSNgB/j4hdB43edfuuwD8Bz1ebfyVwJUCXLr7K08ysUHL5pe0VwOnA0+n0AOA5oIek70fE/bVtKKkF8BDw9YjY6aYpEXEvcC9Av379dhm6wczM8iOXhL8D6BkRqwAkdQR+AfQHZgE1JnxJTUmS/fiI8MiaZmZFkssvbbtWJPvUaqBHRLwHlNW0gSQBY4GXI+Kn9Q/TzMz2Vi4t/D9L+gMwKZ0eAcySdCiwvpZtPg58Hvi7pPnpvP+IiMfqFa2ZmdVbLgn/34BhwJmAgHHAQxERJNfj7yIiZqfrmplZkeUytEJImg1sIxlT54U02ZuZ2X6gzn34kj4LvEDSlfNZ4HlJIwoVmJmZ5VcuXTr/CZxScc29pPYk4+FPLkRgZmaWX7lcpdOo2g+s3s1xezMzK6JcWvh/lPQEMCGdHgn4ahszs/1ELidtvyFpOMmlliIZ+fKRgkVmZmZ5lUsLv+K+tg/tcUUzM9vn7DHh13Brw8pFJFdrtsp7VGZmlnd1GQ/ftzY0MzsA+CobM7OMcMI3M8sIJ3wzs4xwwjczywgnfDOzjHDCNzPLCCd8M7OMcMI3M8sIJ3wzs4xwwjczywgnfDOzjHDCNzPLCCd8M7OMcMI3M8sIJ3wzs4xwwjczywgnfDOzjHDCNzPLCCd8M7OMcMI3M8uIgiZ8SfdJWi1pUSHLMTOzPSt0C/83wPkFLsPMzOqgoAk/ImYB7xWyDDMzq5ui9+FLulLSHElz1qxZU+xwzMwOWEVP+BFxb0T0i4h+7du3L3Y4ZmYHrKInfDMzaxhO+GZmGVHoyzInAH8FjpP0lqQrClmemZnVrkkhdx4Rowq5fzMzqzt36ZiZZYQTvplZRjjhm5llhBO+mVlGOOGbmWWEE76ZWUY44ZuZZYQTvplZRjjhm5llhBO+mVlGOOGbmWWEE76ZWUY44ZuZZYQTvplZRjjhm5llhBO+mVlGOOGbmWWEE76ZWUY44ZuZZYQTvplZRjjhm5llhBO+mVlGOOGbmWWEE76ZWUY44ZuZZYQTvplZRjjhm5llhBO+mVlGOOGbmWVEwRO+pPMlvSJpiaSbCl2emZnVrKAJX1Jj4B7gAuAEYJSkEwpZppmZ1azQLfxTgSUR8XpEbAMmAkMLXKaZmdVAEVG4nUsjgPMj4svp9OeB/hFxdZV1rgSuTCePA14pWED50Q5YW+wgiiTLdYds1z/LdYd9v/5HRUT7Pa3UpMBBqIZ5O33CRMS9wL0FjiNvJM2JiH7FjqMYslx3yHb9s1x3OHDqX+gunbeAI6tMdwZWFLhMMzOrQaET/t+A7pK6SToIuBSYWuAyzcysBgXt0omIcklXA08AjYH7ImJxIctsAPtN91MBZLnukO36Z7nucIDUv6Anbc3MbN85DP9MAAAEx0lEQVThX9qamWWEE76ZWUY44deBpGeLHYPln6TvSrqhHtt9X9InChGTFY6k36S/DSpG2V0lLSpG2VUV+jr8A0JEnFHsGGzfERH/VewYzOrDLfw6kLQ5/TtA0ixJj0h6SdIvJR1Qx1DSo5LmSlqc/goaSZsl3SbpRUkzJLVP58+UdIekZyUtknRqcaPfM0n/mQ7mN53kl91IOkbSH9N6/1nS8ZJaS1pW8fpKai5puaSmVVuKkk5J679A0guSWkpqLOm/Jf1N0kJJ/1rEKteLpOvT13SRpK9LOlTStLSeiySNLHaMeyLpC+nxXyDp/nT22enr9XqV17BF+r5+UdLfJQ1N53eV9LKk/03/H56UdEi6bKakW9LX/FVJZ6Xz9+3XPiL82MMD2Jz+HQBsBY4mucz0T8CIYseX57oenv49BFgEtCX5dfTodP5/AXenz2cC/5s+PxtYVOz491C3vsDfgeZAK2AJcAMwA+iertMfeCp9PgUYmD4fCfwqff4bYARwEPA6cEo6vxXJt+YrgZvTeQcDc4Buxa5/PY7ToUALYDHw7xWvdbpO62LHuYc69CIZpqVdOn14+rpNImnonkAyzhfpa9Yqfd4ufV8I6AqUA6XpsgeBz6XPZwK3pc8vBKanz2t87dN9Ff3/w106uXshIl4HkDQBOBOYXNyQ8uprki5Jnx8JdAd2AA+k834HPFxl/QkAETFLUitJbSJifYNFm5uzgEciYguApKlAM+AMYJJUORLIwenfB0gS/dMkPxr8ebX9HQesjIi/AUTExnS/nwROrtJf3JrkOC4tQJ0K4UyS4/Q+gKSHgTLgE5JuAf4QEX8uZoB1cC4wOSLWAkTEe+nr+2hE7ABektQxXVfAjyWdTfJePwKoWLY0Iuanz+eSJO4KD9cwv7bX/tX8Va3+nPBzV/2HCwfMDxkkDQA+AZweEVskzSRJiNVFLc9rmt7XVI+vEbA+IkprWHcq8BNJh5O0ep+qtlw17K9i/jUR8cTeBlskNY2BBckxuJDkmDwZEd9vwJhyVdtr82G1dQBGA+2BvhFRJmkZH73vq66/neSbb/V9beejXFrjay+pa27hF8YB1f/cQE5Nh4poRNL6m13sgPKoNbAuTfbHA6el8xuRdGEA/DM713kkgKQzgQ0RsaGhgq2HWcAlkg6R1BK4GNgCLJX0GQAlegNExGbgBeBOklbt9mr7+wfQSdIp6bYtJTUh+WX5VyQ1Tef3kHRoA9QvX2YBn07PWxwKXELSit0SEb8D/gfoU8wA62AG8FlJbQHSD+3atAZWp8l+IHDUXpS7T7/2buHn7q/AGOAkkn+MR4obTl79EbhK0kKS/s/n0vnvA70kzQU2kCb51Doll622Ai5vyGBzFREvSnoAmA+8AVR0S4wGfiHpZqApyX0bFqTLHiDp9x1Qw/62pScvf5aezPuA5BvSr0i+4r+opB9hDfDpAlUr79Lj9BuSDztI6tMCeEHSDpLuna8UKbw6iYjFkn4EPCNpOzBvN6uPB/5P0hyS98Y/9qLoffq199AKOUi7PG6IiIuKHUtDkrQ5IlrUMH8myfGY0/BRmVmu3KVjZpYRbuGbmWWEW/hmZhnhhG9mlhFO+GZmGeGEb2aWEU74ZmYZ8f8BeLRexxX8MOcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# visualize number of unique values for each categorical feature\n",
    "x = np.arange(1, 6)\n",
    "unique = [len(talking[i].unique()) for i in heads[:5]]\n",
    "y = np.array([math.log(i, 10) for i in unique])\n",
    "plt.bar(x, y)\n",
    "for i, v in enumerate(unique):\n",
    "    plt.text(i + .8, math.log(v, 10), str(v))\n",
    "\n",
    "plt.ylabel('log(unique_count)')\n",
    "plt.xticks(np.arange(1,6), ('ip', 'app', 'device', 'os', 'channel'))\n",
    "plt.title('Unique Counts for First Five Features')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Down-sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "attr0 = talking[talking['is_attributed']==0]\n",
    "attr1 = talking[talking['is_attributed']==1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "subsample1 = attr0.sample(n=num_attributed_time[1], random_state=3)\n",
    "subsample2 = attr0.sample(n=num_attributed_time[1], random_state=7)\n",
    "subsample3 = attr0.sample(n=num_attributed_time[1], random_state=37)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sub-dataset 1\n",
    "talking1 = pd.concat([attr1, subsample1])\n",
    "# sub-dataset 2\n",
    "talking2 = pd.concat([attr1, subsample2])\n",
    "# sub-dataset 3\n",
    "talking3 = pd.concat([attr1, subsample3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1    456846\n",
      "0    456846\n",
      "Name: is_attributed, dtype: int64\n",
      "1    456846\n",
      "0    456846\n",
      "Name: is_attributed, dtype: int64\n",
      "1    456846\n",
      "0    456846\n",
      "Name: is_attributed, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# Well done! Balanced data now!\n",
    "print(talking1['is_attributed'].value_counts())\n",
    "print(talking2['is_attributed'].value_counts())\n",
    "print(talking3['is_attributed'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save training set into .csv files to avoid loading original huge dataset\n",
    "talking1.to_csv('talking1.csv')\n",
    "talking2.to_csv('talking2.csv')\n",
    "talking3.to_csv('talking3.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load training set directly\n",
    "# talking1 = pd.read_csv('talking1.csv', parse_dates = ['click_time'])\n",
    "# talking2 = pd.read_csv('talking2.csv', parse_dates = ['click_time'])\n",
    "# talking3 = pd.read_csv('talking3.csv', parse_dates = ['click_time'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fit scikit-learn model\n",
    "----"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import LinearSVC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fit 3 Subsamples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# subsample 1\n",
    "x1 = talking1.iloc[:, [0,1,2,3,4,-4,-3,-2,-1]]\n",
    "y1 = talking1.is_attributed\n",
    "X_train_1, X_test_1, y_train_1, y_test_1 = train_test_split(x1, y1, test_size=0.3, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "# subsample 2\n",
    "x2 = talking2.iloc[:, [0,1,2,3,4,-4,-3,-2,-1]]\n",
    "y2 = talking2.is_attributed\n",
    "X_train_2, X_test_2, y_train_2, y_test_2 = train_test_split(x2, y2, test_size=0.3, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "# subsample 3\n",
    "x3 = talking3.iloc[:, [0,1,2,3,4,-4,-3,-2,-1]]\n",
    "y3 = talking3.is_attributed\n",
    "X_train_3, X_test_3, y_train_3, y_test_3 = train_test_split(x3, y3, test_size=0.3, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/envs/ml/lib/python3.6/site-packages/sklearn/svm/base.py:922: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "/anaconda3/envs/ml/lib/python3.6/site-packages/sklearn/svm/base.py:922: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "/anaconda3/envs/ml/lib/python3.6/site-packages/sklearn/svm/base.py:922: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    }
   ],
   "source": [
    "# K-NN\n",
    "knn = neighbors.KNeighborsClassifier()\n",
    "knn_1 = knn.fit(X_train_1, y_train_1)\n",
    "knn_2 = knn.fit(X_train_2, y_train_2)\n",
    "knn_3 = knn.fit(X_train_3, y_train_3)\n",
    "\n",
    "# Logistic Regression\n",
    "logistic = linear_model.LogisticRegression(solver='lbfgs', max_iter=1000,\n",
    "                                           multi_class='multinomial')\n",
    "lr_1 = logistic.fit(X_train_1, y_train_1)\n",
    "lr_2 = logistic.fit(X_train_2, y_train_2)\n",
    "lr_3 = logistic.fit(X_train_3, y_train_3)\n",
    "\n",
    "\n",
    "# Guassian Naive Bayes\n",
    "gnb = GaussianNB()\n",
    "gnb_1 = gnb.fit(X_train_1, y_train_1)\n",
    "gnb_2 = gnb.fit(X_train_2, y_train_2)\n",
    "gnb_3 = gnb.fit(X_train_3, y_train_3)\n",
    "\n",
    "\n",
    "# Support Vector Classifier\n",
    "svc = LinearSVC(C=1.0)\n",
    "svc_1 = svc.fit(X_train_1, y_train_1)\n",
    "svc_2 = svc.fit(X_train_2, y_train_2)\n",
    "svc_3 = svc.fit(X_train_3, y_train_3)\n",
    "\n",
    "\n",
    "# Random Forest Classifier\n",
    "rfc = RandomForestClassifier(n_estimators=100)\n",
    "rfc_1 = rfc.fit(X_train_1, y_train_1)\n",
    "rfc_2 = rfc.fit(X_train_2, y_train_2)\n",
    "rfc_3 = rfc.fit(X_train_3, y_train_3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluation Metric\n",
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Subsample 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KNN score: 0.809984\n",
      "LogisticRegression score: 0.752131\n",
      "gnb score: 0.747658\n",
      "svc score: 0.505454\n",
      "rfc score: 0.920918\n"
     ]
    }
   ],
   "source": [
    "knn_1_score = knn_1.score(X_test_1, y_test_1)\n",
    "lr_1_score = lr_1.score(X_test_1, y_test_1)\n",
    "gnb_1_score = gnb_1.score(X_test_1, y_test_1)\n",
    "svc_1_score = svc_1.score(X_test_1, y_test_1)\n",
    "rfc_1_score = rfc_1.score(X_test_1, y_test_1)\n",
    "\n",
    "print('KNN score: %f' % knn_1_score)\n",
    "print('LogisticRegression score: %f' % lr_1_score)\n",
    "print('gnb score: %f' % gnb_1_score)\n",
    "print('svc score: %f' % svc_1_score)\n",
    "print('rfc score: %f' % rfc_1_score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Subsample 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KNN score: 0.810057\n",
      "LogisticRegression score: 0.752069\n",
      "gnb score: 0.747180\n",
      "svc score: 0.505866\n",
      "rfc score: 0.920400\n"
     ]
    }
   ],
   "source": [
    "knn_2_score = knn_2.score(X_test_2, y_test_2)\n",
    "lr_2_score = lr_2.score(X_test_2, y_test_2)\n",
    "gnb_2_score = gnb_2.score(X_test_2, y_test_2)\n",
    "svc_2_score = svc_2.score(X_test_2, y_test_2)\n",
    "rfc_2_score = rfc_2.score(X_test_2, y_test_2)\n",
    "\n",
    "print('KNN score: %f' % knn_2_score)\n",
    "print('LogisticRegression score: %f' % lr_2_score)\n",
    "print('gnb score: %f' % gnb_2_score)\n",
    "print('svc score: %f' % svc_2_score)\n",
    "print('rfc score: %f' % rfc_2_score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Subsample 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KNN score: 0.809721\n",
      "LogisticRegression score: 0.752966\n",
      "gnb score: 0.747282\n",
      "svc score: 0.505870\n",
      "rfc score: 0.920721\n"
     ]
    }
   ],
   "source": [
    "knn_3_score = knn_3.score(X_test_3, y_test_3)\n",
    "lr_3_score = lr_3.score(X_test_3, y_test_3)\n",
    "gnb_3_score = gnb_3.score(X_test_3, y_test_3)\n",
    "svc_3_score = svc_3.score(X_test_3, y_test_3)\n",
    "rfc_3_score = rfc_3.score(X_test_3, y_test_3)\n",
    "\n",
    "print('KNN score: %f' % knn_3_score)\n",
    "print('LogisticRegression score: %f' % lr_3_score)\n",
    "print('gnb score: %f' % gnb_3_score)\n",
    "print('svc score: %f' % svc_3_score)\n",
    "print('rfc score: %f' % rfc_3_score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Average Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average KNN score: 0.809920\n",
      "Average LogisticRegression score: 0.752388\n",
      "Average Gaussian Naive Bayes score: 0.747373\n",
      "Average SVC score: 0.505730\n",
      "Average RFC score: 0.920680\n"
     ]
    }
   ],
   "source": [
    "print('Average KNN score: %f' % ((knn_1_score + knn_2_score + knn_3_score) / 3))\n",
    "print('Average LogisticRegression score: %f' % ((lr_1_score + lr_2_score + lr_3_score) / 3))\n",
    "print('Average Gaussian Naive Bayes score: %f' % ((gnb_1_score + gnb_2_score + gnb_3_score) / 3))\n",
    "print('Average SVC score: %f' % ((svc_1_score + svc_2_score + svc_3_score) / 3))\n",
    "print('Average RFC score: %f' % ((rfc_1_score + rfc_2_score + rfc_3_score) / 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}